xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/refact/tsdb_optimize' into refact/tsdb_optimize 

# Conflicts:
#	source/dnode/vnode/src/inc/tsdb.h
#	source/dnode/vnode/src/tsdb/tsdbMergeTree.c
#	source/dnode/vnode/src/tsdb/tsdbRead.c
This commit is contained in:
Haojun Liao 2022-09-02 11:23:01 +08:00
parent 720f3b2482 0eb29455ae
commit cac1fd36b7
12 changed files with 533 additions and 547 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
source/common/src/tglobal.c
View File

@ -129,10 +129,6 @@ int32_t tsMinIntervalTime = 1;
int32_t tsQueryBufferSize = -1; int32_t tsQueryBufferSize = -1;
int64_t tsQueryBufferSizeBytes = -1; int64_t tsQueryBufferSizeBytes = -1;
// tsdb config
// For backward compatibility
bool tsdbForceKeepFile = false;
int32_t tsDiskCfgNum = 0; int32_t tsDiskCfgNum = 0;
SDiskCfg tsDiskCfg[TFS_MAX_DISKS] = {0}; SDiskCfg tsDiskCfg[TFS_MAX_DISKS] = {0};

70
source/dnode/vnode/src/inc/tsdb.h
View File

@ -42,15 +42,15 @@ typedef struct SMemTable SMemTable;
typedef struct STbDataIter STbDataIter; typedef struct STbDataIter STbDataIter;
typedef struct SMapData SMapData; typedef struct SMapData SMapData;
typedef struct SBlockIdx SBlockIdx; typedef struct SBlockIdx SBlockIdx;
typedef struct SBlock SBlock; typedef struct SDataBlk SDataBlk;
typedef struct SBlockL SBlockL; typedef struct SSstBlk SSstBlk;
typedef struct SColData SColData; typedef struct SColData SColData;
typedef struct SDiskDataHdr SDiskDataHdr; typedef struct SDiskDataHdr SDiskDataHdr;
typedef struct SBlockData SBlockData; typedef struct SBlockData SBlockData;
typedef struct SDelFile SDelFile; typedef struct SDelFile SDelFile;
typedef struct SHeadFile SHeadFile; typedef struct SHeadFile SHeadFile;
typedef struct SDataFile SDataFile; typedef struct SDataFile SDataFile;
typedef struct SLastFile SLastFile; typedef struct SSstFile SSstFile;
typedef struct SSmaFile SSmaFile; typedef struct SSmaFile SSmaFile;
typedef struct SDFileSet SDFileSet; typedef struct SDFileSet SDFileSet;
typedef struct SDataFWriter SDataFWriter; typedef struct SDataFWriter SDataFWriter;
@ -114,15 +114,15 @@ int32_t tTABLEIDCmprFn(const void *p1, const void *p2);
int32_t tPutBlockCol(uint8_t *p, void *ph); int32_t tPutBlockCol(uint8_t *p, void *ph);
int32_t tGetBlockCol(uint8_t *p, void *ph); int32_t tGetBlockCol(uint8_t *p, void *ph);
int32_t tBlockColCmprFn(const void *p1, const void *p2); int32_t tBlockColCmprFn(const void *p1, const void *p2);
// SBlock // SDataBlk
void tBlockReset(SBlock *pBlock); void tDataBlkReset(SDataBlk *pBlock);
int32_t tPutBlock(uint8_t *p, void *ph); int32_t tPutDataBlk(uint8_t *p, void *ph);
int32_t tGetBlock(uint8_t *p, void *ph); int32_t tGetDataBlk(uint8_t *p, void *ph);
int32_t tBlockCmprFn(const void *p1, const void *p2); int32_t tDataBlkCmprFn(const void *p1, const void *p2);
bool tBlockHasSma(SBlock *pBlock); bool tDataBlkHasSma(SDataBlk *pDataBlk);
// SBlockL // SSstBlk
int32_t tPutBlockL(uint8_t *p, void *ph); int32_t tPutSstBlk(uint8_t *p, void *ph);
int32_t tGetBlockL(uint8_t *p, void *ph); int32_t tGetSstBlk(uint8_t *p, void *ph);
// SBlockIdx // SBlockIdx
int32_t tPutBlockIdx(uint8_t *p, void *ph); int32_t tPutBlockIdx(uint8_t *p, void *ph);
int32_t tGetBlockIdx(uint8_t *p, void *ph); int32_t tGetBlockIdx(uint8_t *p, void *ph);
@ -219,7 +219,7 @@ bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2);
int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype); int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype);
int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile); int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile);
int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile); int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile);
int32_t tPutLastFile(uint8_t *p, SLastFile *pLastFile); int32_t tPutSstFile(uint8_t *p, SSstFile *pSstFile);
int32_t tPutSmaFile(uint8_t *p, SSmaFile *pSmaFile); int32_t tPutSmaFile(uint8_t *p, SSmaFile *pSmaFile);
int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile); int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile);
int32_t tGetDelFile(uint8_t *p, SDelFile *pDelFile); int32_t tGetDelFile(uint8_t *p, SDelFile *pDelFile);
@ -228,7 +228,7 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet);
void tsdbHeadFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SHeadFile *pHeadF, char fname[]); void tsdbHeadFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SHeadFile *pHeadF, char fname[]);
void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF, char fname[]); void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF, char fname[]);
void tsdbLastFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SLastFile *pLastF, char fname[]); void tsdbSstFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSstFile *pSstF, char fname[]);
void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]); void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]);
// SDelFile // SDelFile
void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]); void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]);
@ -254,7 +254,7 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync);
int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter); int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter);
int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx); int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx);
int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, SBlockIdx *pBlockIdx); int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, SBlockIdx *pBlockIdx);
int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL); int32_t tsdbWriteSstBlk(SDataFWriter *pWriter, SArray *aSstBlk);
int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo,
int8_t cmprAlg, int8_t toLast); int8_t cmprAlg, int8_t toLast);
@ -264,10 +264,10 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
int32_t tsdbDataFReaderClose(SDataFReader **ppReader); int32_t tsdbDataFReaderClose(SDataFReader **ppReader);
int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx); int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx);
int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData); int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData);
int32_t tsdbReadBlockL(SDataFReader *pReader, int32_t iLast, SArray *aBlockL); int32_t tsdbReadSstBlk(SDataFReader *pReader, int32_t iSst, SArray *aSstBlk);
int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg); int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pBlock, SArray *aColumnDataAgg);
int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData); int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pBlock, SBlockData *pBlockData);
int32_t tsdbReadLastBlock(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, SBlockData *pBlockData); int32_t tsdbReadSstBlock(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData);
// SDelFWriter // SDelFWriter
int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb);
int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync); int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync);
@ -427,7 +427,7 @@ struct SSmaInfo {
int32_t size; int32_t size;
}; };
struct SBlock { struct SDataBlk {
TSDBKEY minKey; TSDBKEY minKey;
TSDBKEY maxKey; TSDBKEY maxKey;
int64_t minVer; int64_t minVer;
@ -439,7 +439,7 @@ struct SBlock {
SSmaInfo smaInfo; SSmaInfo smaInfo;
}; };
struct SBlockL { struct SSstBlk {
int64_t suid; int64_t suid;
int64_t minUid; int64_t minUid;
int64_t maxUid; int64_t maxUid;
@ -478,12 +478,6 @@ struct SBlockData {
SArray *aColData; // SArray<SColData> SArray *aColData; // SArray<SColData>
}; };
// ================== TSDB global config
extern bool tsdbForceKeepFile;
#define TSDB_FS_ITER_FORWARD TSDB_ORDER_ASC
#define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC
struct TABLEID { struct TABLEID {
tb_uid_t suid; tb_uid_t suid;
tb_uid_t uid; tb_uid_t uid;
@ -547,7 +541,7 @@ struct SDataFile {
int64_t size; int64_t size;
}; };
struct SLastFile { struct SSstFile {
volatile int32_t nRef; volatile int32_t nRef;
int64_t commitID; int64_t commitID;
@ -568,8 +562,8 @@ struct SDFileSet {
SHeadFile *pHeadF; SHeadFile *pHeadF;
SDataFile *pDataF; SDataFile *pDataF;
SSmaFile *pSmaF; SSmaFile *pSmaF;
uint8_t nLastF; uint8_t nSstF;
SLastFile *aLastF[TSDB_MAX_LAST_FILE]; SSstFile *aSstF[TSDB_MAX_LAST_FILE];
}; };
struct SRowIter { struct SRowIter {
@ -604,7 +598,7 @@ struct SDataFWriter {
SHeadFile fHead; SHeadFile fHead;
SDataFile fData; SDataFile fData;
SSmaFile fSma; SSmaFile fSma;
SLastFile fLast[TSDB_MAX_LAST_FILE]; SSstFile fSst[TSDB_MAX_LAST_FILE];
uint8_t *aBuf[4]; uint8_t *aBuf[4];
}; };
@ -633,17 +627,17 @@ typedef struct {
} SRowInfo; } SRowInfo;
typedef struct SMergeTree { typedef struct SMergeTree {
int8_t backward; int8_t backward;
SRBTree rbt; SRBTree rbt;
SArray *pIterList; SArray *pIterList;
struct SLDataIter *pIter; struct SLDataIter *pIter;
} SMergeTree; } SMergeTree;
int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, uint64_t uid, STimeWindow* pTimeWindow, SVersionRange* pVerRange); int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, uint64_t uid, STimeWindow* pTimeWindow, SVersionRange* pVerRange);
void tMergeTreeAddIter(SMergeTree *pMTree, struct SLDataIter *pIter); void tMergeTreeAddIter(SMergeTree *pMTree, struct SLDataIter *pIter);
bool tMergeTreeNext(SMergeTree* pMTree); bool tMergeTreeNext(SMergeTree *pMTree);
TSDBROW tMergeTreeGetRow(SMergeTree* pMTree); TSDBROW tMergeTreeGetRow(SMergeTree *pMTree);
void tMergeTreeClose(SMergeTree* pMTree); void tMergeTreeClose(SMergeTree *pMTree);
// ========== inline functions ========== // ========== inline functions ==========
static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) {

8
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -527,7 +527,7 @@ typedef struct SFSNextRowIter {
SMapData blockMap; SMapData blockMap;
int32_t nBlock; int32_t nBlock;
int32_t iBlock; int32_t iBlock;
SBlock block; SDataBlk block;
SBlockData blockData; SBlockData blockData;
SBlockData *pBlockData; SBlockData *pBlockData;
int32_t nRow; int32_t nRow;
@ -602,13 +602,13 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
} }
case SFSNEXTROW_BLOCKDATA: case SFSNEXTROW_BLOCKDATA:
if (state->iBlock >= 0) { if (state->iBlock >= 0) {
SBlock block = {0}; SDataBlk block = {0};
tBlockReset(&block); tDataBlkReset(&block);
// tBlockDataReset(&state->blockData); // tBlockDataReset(&state->blockData);
tBlockDataReset(state->pBlockData); tBlockDataReset(state->pBlockData);
tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetBlock); tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetDataBlk);
/* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); */ /* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); */
tBlockDataReset(state->pBlockData); tBlockDataReset(state->pBlockData);
code = tBlockDataInit(state->pBlockData, state->suid, state->uid, state->pTSchema); code = tBlockDataInit(state->pBlockData, state->suid, state->uid, state->pTSchema);

150
source/dnode/vnode/src/tsdb/tsdbCommit.c
View File

@ -32,12 +32,12 @@ typedef struct {
STbDataIter iter; STbDataIter iter;
}; // memory data iter }; // memory data iter
struct { struct {
int32_t iLast; int32_t iSst;
SArray *aBlockL; SArray *aSstBlk;
int32_t iBlockL; int32_t iSstBlk;
SBlockData bData; SBlockData bData;
int32_t iRow; int32_t iRow;
}; // last file data iter }; // sst file data iter
}; };
} SDataIter; } SDataIter;
@ -64,7 +64,7 @@ typedef struct {
SArray *aBlockIdx; // SArray<SBlockIdx> SArray *aBlockIdx; // SArray<SBlockIdx>
int32_t iBlockIdx; int32_t iBlockIdx;
SBlockIdx *pBlockIdx; SBlockIdx *pBlockIdx;
SMapData mBlock; // SMapData<SBlock> SMapData mBlock; // SMapData<SDataBlk>
SBlockData bData; SBlockData bData;
} dReader; } dReader;
struct { struct {
@ -77,8 +77,8 @@ typedef struct {
struct { struct {
SDataFWriter *pWriter; SDataFWriter *pWriter;
SArray *aBlockIdx; // SArray<SBlockIdx> SArray *aBlockIdx; // SArray<SBlockIdx>
SArray *aBlockL; // SArray<SBlockL> SArray *aSstBlk; // SArray<SSstBlk>
SMapData mBlock; // SMapData<SBlock> SMapData mBlock; // SMapData<SDataBlk>
SBlockData bData; SBlockData bData;
SBlockData bDatal; SBlockData bDatal;
} dWriter; } dWriter;
@ -92,8 +92,8 @@ typedef struct {
SArray *aDelData; // SArray<SDelData> SArray *aDelData; // SArray<SDelData>
} SCommitter; } SCommitter;
extern int32_t tsdbReadLastBlockEx(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, extern int32_t tsdbReadSstBlockEx(SDataFReader *pReader, int32_t iSst, SSstBlk *aSstBlk,
SBlockData *pBlockData); // todo SBlockData *pBlockData); // todo
static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter); static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter);
static int32_t tsdbCommitData(SCommitter *pCommitter); static int32_t tsdbCommitData(SCommitter *pCommitter);
@ -431,21 +431,21 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) {
pCommitter->toLastOnly = 0; pCommitter->toLastOnly = 0;
SDataFReader *pReader = pCommitter->dReader.pReader; SDataFReader *pReader = pCommitter->dReader.pReader;
if (pReader) { if (pReader) {
if (pReader->pSet->nLastF >= pCommitter->maxLast) { if (pReader->pSet->nSstF >= pCommitter->maxLast) {
int8_t iIter = 0; int8_t iIter = 0;
for (int32_t iLast = 0; iLast < pReader->pSet->nLastF; iLast++) { for (int32_t iSst = 0; iSst < pReader->pSet->nSstF; iSst++) {
pIter = &pCommitter->aDataIter[iIter]; pIter = &pCommitter->aDataIter[iIter];
pIter->type = LAST_DATA_ITER; pIter->type = LAST_DATA_ITER;
pIter->iLast = iLast; pIter->iSst = iSst;
code = tsdbReadBlockL(pCommitter->dReader.pReader, iLast, pIter->aBlockL); code = tsdbReadSstBlk(pCommitter->dReader.pReader, iSst, pIter->aSstBlk);
if (code) goto _err; if (code) goto _err;
if (taosArrayGetSize(pIter->aBlockL) == 0) continue; if (taosArrayGetSize(pIter->aSstBlk) == 0) continue;
pIter->iBlockL = 0; pIter->iSstBlk = 0;
SBlockL *pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, 0); SSstBlk *pSstBlk = (SSstBlk *)taosArrayGet(pIter->aSstBlk, 0);
code = tsdbReadLastBlockEx(pCommitter->dReader.pReader, iLast, pBlockL, &pIter->bData); code = tsdbReadSstBlockEx(pCommitter->dReader.pReader, iSst, pSstBlk, &pIter->bData);
if (code) goto _err; if (code) goto _err;
pIter->iRow = 0; pIter->iRow = 0;
@ -457,9 +457,9 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) {
iIter++; iIter++;
} }
} else { } else {
for (int32_t iLast = 0; iLast < pReader->pSet->nLastF; iLast++) { for (int32_t iSst = 0; iSst < pReader->pSet->nSstF; iSst++) {
SLastFile *pLastFile = pReader->pSet->aLastF[iLast]; SSstFile *pSstFile = pReader->pSet->aSstF[iSst];
if (pLastFile->size > pLastFile->offset) { if (pSstFile->size > pSstFile->offset) {
pCommitter->toLastOnly = 1; pCommitter->toLastOnly = 1;
break; break;
} }
@ -515,34 +515,34 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
SHeadFile fHead = {.commitID = pCommitter->commitID}; SHeadFile fHead = {.commitID = pCommitter->commitID};
SDataFile fData = {.commitID = pCommitter->commitID}; SDataFile fData = {.commitID = pCommitter->commitID};
SSmaFile fSma = {.commitID = pCommitter->commitID}; SSmaFile fSma = {.commitID = pCommitter->commitID};
SLastFile fLast = {.commitID = pCommitter->commitID}; SSstFile fSst = {.commitID = pCommitter->commitID};
SDFileSet wSet = {.fid = pCommitter->commitFid, .pHeadF = &fHead, .pDataF = &fData, .pSmaF = &fSma}; SDFileSet wSet = {.fid = pCommitter->commitFid, .pHeadF = &fHead, .pDataF = &fData, .pSmaF = &fSma};
if (pRSet) { if (pRSet) {
ASSERT(pRSet->nLastF <= pCommitter->maxLast); ASSERT(pRSet->nSstF <= pCommitter->maxLast);
fData = *pRSet->pDataF; fData = *pRSet->pDataF;
fSma = *pRSet->pSmaF; fSma = *pRSet->pSmaF;
wSet.diskId = pRSet->diskId; wSet.diskId = pRSet->diskId;
if (pRSet->nLastF < pCommitter->maxLast) { if (pRSet->nSstF < pCommitter->maxLast) {
for (int32_t iLast = 0; iLast < pRSet->nLastF; iLast++) { for (int32_t iSst = 0; iSst < pRSet->nSstF; iSst++) {
wSet.aLastF[iLast] = pRSet->aLastF[iLast]; wSet.aSstF[iSst] = pRSet->aSstF[iSst];
} }
wSet.nLastF = pRSet->nLastF + 1; wSet.nSstF = pRSet->nSstF + 1;
} else { } else {
wSet.nLastF = 1; wSet.nSstF = 1;
} }
} else { } else {
SDiskID did = {0}; SDiskID did = {0};
tfsAllocDisk(pTsdb->pVnode->pTfs, 0, &did); tfsAllocDisk(pTsdb->pVnode->pTfs, 0, &did);
tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did); tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did);
wSet.diskId = did; wSet.diskId = did;
wSet.nLastF = 1; wSet.nSstF = 1;
} }
wSet.aLastF[wSet.nLastF - 1] = &fLast; wSet.aSstF[wSet.nSstF - 1] = &fSst;
code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet); code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet);
if (code) goto _err; if (code) goto _err;
taosArrayClear(pCommitter->dWriter.aBlockIdx); taosArrayClear(pCommitter->dWriter.aBlockIdx);
taosArrayClear(pCommitter->dWriter.aBlockL); taosArrayClear(pCommitter->dWriter.aSstBlk);
tMapDataReset(&pCommitter->dWriter.mBlock); tMapDataReset(&pCommitter->dWriter.mBlock);
tBlockDataReset(&pCommitter->dWriter.bData); tBlockDataReset(&pCommitter->dWriter.bData);
tBlockDataReset(&pCommitter->dWriter.bDatal); tBlockDataReset(&pCommitter->dWriter.bDatal);
@ -562,11 +562,11 @@ _err:
static int32_t tsdbCommitDataBlock(SCommitter *pCommitter) { static int32_t tsdbCommitDataBlock(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
SBlockData *pBlockData = &pCommitter->dWriter.bData; SBlockData *pBlockData = &pCommitter->dWriter.bData;
SBlock block; SDataBlk block;
ASSERT(pBlockData->nRow > 0); ASSERT(pBlockData->nRow > 0);
tBlockReset(&block); tDataBlkReset(&block);
// info // info
block.nRow += pBlockData->nRow; block.nRow += pBlockData->nRow;
@ -597,8 +597,8 @@ static int32_t tsdbCommitDataBlock(SCommitter *pCommitter) {
((block.nSubBlock == 1) && !block.hasDup) ? &block.smaInfo : NULL, pCommitter->cmprAlg, 0); ((block.nSubBlock == 1) && !block.hasDup) ? &block.smaInfo : NULL, pCommitter->cmprAlg, 0);
if (code) goto _err; if (code) goto _err;
// put SBlock // put SDataBlk
code = tMapDataPutItem(&pCommitter->dWriter.mBlock, &block, tPutBlock); code = tMapDataPutItem(&pCommitter->dWriter.mBlock, &block, tPutDataBlk);
if (code) goto _err; if (code) goto _err;
// clear // clear
@ -613,7 +613,7 @@ _err:
static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
SBlockL blockL; SSstBlk blockL;
SBlockData *pBlockData = &pCommitter->dWriter.bDatal; SBlockData *pBlockData = &pCommitter->dWriter.bDatal;
ASSERT(pBlockData->nRow > 0); ASSERT(pBlockData->nRow > 0);
@ -638,8 +638,8 @@ static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) {
code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &blockL.bInfo, NULL, pCommitter->cmprAlg, 1); code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &blockL.bInfo, NULL, pCommitter->cmprAlg, 1);
if (code) goto _err; if (code) goto _err;
// push SBlockL // push SSstBlk
if (taosArrayPush(pCommitter->dWriter.aBlockL, &blockL) == NULL) { if (taosArrayPush(pCommitter->dWriter.aSstBlk, &blockL) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
@ -661,8 +661,8 @@ static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) {
code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx); code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx);
if (code) goto _err; if (code) goto _err;
// write aBlockL // write aSstBlk
code = tsdbWriteBlockL(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockL); code = tsdbWriteSstBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.aSstBlk);
if (code) goto _err; if (code) goto _err;
// update file header // update file header
@ -790,10 +790,10 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) {
if (code) goto _exit; if (code) goto _exit;
// merger // merger
for (int32_t iLast = 0; iLast < TSDB_MAX_LAST_FILE; iLast++) { for (int32_t iSst = 0; iSst < TSDB_MAX_LAST_FILE; iSst++) {
SDataIter *pIter = &pCommitter->aDataIter[iLast]; SDataIter *pIter = &pCommitter->aDataIter[iSst];
pIter->aBlockL = taosArrayInit(0, sizeof(SBlockL)); pIter->aSstBlk = taosArrayInit(0, sizeof(SSstBlk));
if (pIter->aBlockL == NULL) { if (pIter->aSstBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
@ -809,8 +809,8 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) {
goto _exit; goto _exit;
} }
pCommitter->dWriter.aBlockL = taosArrayInit(0, sizeof(SBlockL)); pCommitter->dWriter.aSstBlk = taosArrayInit(0, sizeof(SSstBlk));
if (pCommitter->dWriter.aBlockL == NULL) { if (pCommitter->dWriter.aSstBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
@ -832,15 +832,15 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) {
tBlockDataDestroy(&pCommitter->dReader.bData, 1); tBlockDataDestroy(&pCommitter->dReader.bData, 1);
// merger // merger
for (int32_t iLast = 0; iLast < TSDB_MAX_LAST_FILE; iLast++) { for (int32_t iSst = 0; iSst < TSDB_MAX_LAST_FILE; iSst++) {
SDataIter *pIter = &pCommitter->aDataIter[iLast]; SDataIter *pIter = &pCommitter->aDataIter[iSst];
taosArrayDestroy(pIter->aBlockL); taosArrayDestroy(pIter->aSstBlk);
tBlockDataDestroy(&pIter->bData, 1); tBlockDataDestroy(&pIter->bData, 1);
} }
// writer // writer
taosArrayDestroy(pCommitter->dWriter.aBlockIdx); taosArrayDestroy(pCommitter->dWriter.aBlockIdx);
taosArrayDestroy(pCommitter->dWriter.aBlockL); taosArrayDestroy(pCommitter->dWriter.aSstBlk);
tMapDataClear(&pCommitter->dWriter.mBlock); tMapDataClear(&pCommitter->dWriter.mBlock);
tBlockDataDestroy(&pCommitter->dWriter.bData, 1); tBlockDataDestroy(&pCommitter->dWriter.bData, 1);
tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1); tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1);
@ -1055,11 +1055,11 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) {
pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow]; pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow];
pIter->r.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); pIter->r.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow);
} else { } else {
pIter->iBlockL++; pIter->iSstBlk++;
if (pIter->iBlockL < taosArrayGetSize(pIter->aBlockL)) { if (pIter->iSstBlk < taosArrayGetSize(pIter->aSstBlk)) {
SBlockL *pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, pIter->iBlockL); SSstBlk *pSstBlk = (SSstBlk *)taosArrayGet(pIter->aSstBlk, pIter->iSstBlk);
code = tsdbReadLastBlockEx(pCommitter->dReader.pReader, pIter->iLast, pBlockL, &pIter->bData); code = tsdbReadSstBlockEx(pCommitter->dReader.pReader, pIter->iSst, pSstBlk, &pIter->bData);
if (code) goto _exit; if (code) goto _exit;
pIter->iRow = 0; pIter->iRow = 0;
@ -1098,7 +1098,7 @@ _exit:
return code; return code;
} }
static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SBlock *pBlock) { static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) {
int32_t code = 0; int32_t code = 0;
SBlockData *pBlockData = &pCommitter->dWriter.bData; SBlockData *pBlockData = &pCommitter->dWriter.bData;
SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter);
@ -1122,7 +1122,7 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SBlock *pBlock) {
pRowInfo = NULL; pRowInfo = NULL;
} else { } else {
TSDBKEY tKey = TSDBROW_KEY(&pRowInfo->row); TSDBKEY tKey = TSDBROW_KEY(&pRowInfo->row);
if (tsdbKeyCmprFn(&tKey, &pBlock->minKey) >= 0) pRowInfo = NULL; if (tsdbKeyCmprFn(&tKey, &pDataBlk->minKey) >= 0) pRowInfo = NULL;
} }
} }
@ -1144,14 +1144,14 @@ _err:
return code; return code;
} }
static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SBlock *pBlock) { static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) {
int32_t code = 0; int32_t code = 0;
SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter);
TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid}; TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid};
SBlockData *pBDataR = &pCommitter->dReader.bData; SBlockData *pBDataR = &pCommitter->dReader.bData;
SBlockData *pBDataW = &pCommitter->dWriter.bData; SBlockData *pBDataW = &pCommitter->dWriter.bData;
code = tsdbReadDataBlock(pCommitter->dReader.pReader, pBlock, pBDataR); code = tsdbReadDataBlock(pCommitter->dReader.pReader, pDataBlk, pBDataR);
if (code) goto _err; if (code) goto _err;
tBlockDataClear(pBDataW); tBlockDataClear(pBDataW);
@ -1188,7 +1188,7 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SBlock *pBlock) {
pRowInfo = NULL; pRowInfo = NULL;
} else { } else {
TSDBKEY tKey = TSDBROW_KEY(&pRowInfo->row); TSDBKEY tKey = TSDBROW_KEY(&pRowInfo->row);
if (tsdbKeyCmprFn(&tKey, &pBlock->maxKey) > 0) pRowInfo = NULL; if (tsdbKeyCmprFn(&tKey, &pDataBlk->maxKey) > 0) pRowInfo = NULL;
} }
} }
} else { } else {
@ -1237,57 +1237,57 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) {
ASSERT(pBlockIdx == NULL || tTABLEIDCmprFn(pBlockIdx, &id) >= 0); ASSERT(pBlockIdx == NULL || tTABLEIDCmprFn(pBlockIdx, &id) >= 0);
if (pBlockIdx && pBlockIdx->suid == id.suid && pBlockIdx->uid == id.uid) { if (pBlockIdx && pBlockIdx->suid == id.suid && pBlockIdx->uid == id.uid) {
int32_t iBlock = 0; int32_t iBlock = 0;
SBlock block; SDataBlk block;
SBlock *pBlock = &block; SDataBlk *pDataBlk = &block;
SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter);
ASSERT(pRowInfo->suid == id.suid && pRowInfo->uid == id.uid); ASSERT(pRowInfo->suid == id.suid && pRowInfo->uid == id.uid);
tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk);
while (pBlock && pRowInfo) { while (pDataBlk && pRowInfo) {
SBlock tBlock = {.minKey = TSDBROW_KEY(&pRowInfo->row), .maxKey = TSDBROW_KEY(&pRowInfo->row)}; SDataBlk tBlock = {.minKey = TSDBROW_KEY(&pRowInfo->row), .maxKey = TSDBROW_KEY(&pRowInfo->row)};
int32_t c = tBlockCmprFn(pBlock, &tBlock); int32_t c = tDataBlkCmprFn(pDataBlk, &tBlock);
if (c < 0) { if (c < 0) {
code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock); code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk);
if (code) goto _err; if (code) goto _err;
iBlock++; iBlock++;
if (iBlock < pCommitter->dReader.mBlock.nItem) { if (iBlock < pCommitter->dReader.mBlock.nItem) {
tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk);
} else { } else {
pBlock = NULL; pDataBlk = NULL;
} }
} else if (c > 0) { } else if (c > 0) {
code = tsdbCommitAheadBlock(pCommitter, pBlock); code = tsdbCommitAheadBlock(pCommitter, pDataBlk);
if (code) goto _err; if (code) goto _err;
pRowInfo = tsdbGetCommitRow(pCommitter); pRowInfo = tsdbGetCommitRow(pCommitter);
if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL; if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL;
} else { } else {
code = tsdbCommitMergeBlock(pCommitter, pBlock); code = tsdbCommitMergeBlock(pCommitter, pDataBlk);
if (code) goto _err; if (code) goto _err;
iBlock++; iBlock++;
if (iBlock < pCommitter->dReader.mBlock.nItem) { if (iBlock < pCommitter->dReader.mBlock.nItem) {
tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk);
} else { } else {
pBlock = NULL; pDataBlk = NULL;
} }
pRowInfo = tsdbGetCommitRow(pCommitter); pRowInfo = tsdbGetCommitRow(pCommitter);
if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL; if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL;
} }
} }
while (pBlock) { while (pDataBlk) {
code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock); code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk);
if (code) goto _err; if (code) goto _err;
iBlock++; iBlock++;
if (iBlock < pCommitter->dReader.mBlock.nItem) { if (iBlock < pCommitter->dReader.mBlock.nItem) {
tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk);
} else { } else {
pBlock = NULL; pDataBlk = NULL;
} }
} }

192
source/dnode/vnode/src/tsdb/tsdbFS.c
View File

@ -110,7 +110,7 @@ _err:
// taosRemoveFile(fname); // taosRemoveFile(fname);
// } // }
// // last // // sst
// if (isSameDisk && pFrom->pLastF->commitID == pTo->pLastF->commitID) { // if (isSameDisk && pFrom->pLastF->commitID == pTo->pLastF->commitID) {
// if (pFrom->pLastF->size > pTo->pLastF->size) { // if (pFrom->pLastF->size > pTo->pLastF->size) {
// code = tsdbDFileRollback(pFS->pTsdb, pTo, TSDB_LAST_FILE); // code = tsdbDFileRollback(pFS->pTsdb, pTo, TSDB_LAST_FILE);
@ -140,7 +140,7 @@ _err:
// tsdbDataFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pDataF, fname); // tsdbDataFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pDataF, fname);
// taosRemoveFile(fname); // taosRemoveFile(fname);
// // last // // sst
// tsdbLastFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pLastF, fname); // tsdbLastFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pLastF, fname);
// taosRemoveFile(fname); // taosRemoveFile(fname);
@ -255,8 +255,8 @@ void tsdbFSDestroy(STsdbFS *pFS) {
taosMemoryFree(pSet->pHeadF); taosMemoryFree(pSet->pHeadF);
taosMemoryFree(pSet->pDataF); taosMemoryFree(pSet->pDataF);
taosMemoryFree(pSet->pSmaF); taosMemoryFree(pSet->pSmaF);
for (int32_t iLast = 0; iLast < pSet->nLastF; iLast++) { for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
taosMemoryFree(pSet->aLastF[iLast]); taosMemoryFree(pSet->aSstF[iSst]);
} }
} }
@ -311,13 +311,13 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
if (code) goto _err; if (code) goto _err;
} }
// last =========== // sst ===========
tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aLastF[0], fname); tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[0], fname);
if (taosStatFile(fname, &size, NULL)) { if (taosStatFile(fname, &size, NULL)) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
if (size != pSet->aLastF[0]->size) { if (size != pSet->aSstF[0]->size) {
code = TSDB_CODE_FILE_CORRUPTED; code = TSDB_CODE_FILE_CORRUPTED;
goto _err; goto _err;
} }
@ -508,13 +508,15 @@ int32_t tsdbFSClose(STsdb *pTsdb) {
ASSERT(pSet->pDataF->nRef == 1); ASSERT(pSet->pDataF->nRef == 1);
taosMemoryFree(pSet->pDataF); taosMemoryFree(pSet->pDataF);
// last
ASSERT(pSet->aLastF[0]->nRef == 1);
taosMemoryFree(pSet->aLastF[0]);
// sma // sma
ASSERT(pSet->pSmaF->nRef == 1); ASSERT(pSet->pSmaF->nRef == 1);
taosMemoryFree(pSet->pSmaF); taosMemoryFree(pSet->pSmaF);
// sst
for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
ASSERT(pSet->aSstF[iSst]->nRef == 1);
taosMemoryFree(pSet->aSstF[iSst]);
}
} }
taosArrayDestroy(pTsdb->fs.aDFileSet); taosArrayDestroy(pTsdb->fs.aDFileSet);
@ -570,14 +572,14 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) {
} }
*fSet.pSmaF = *pSet->pSmaF; *fSet.pSmaF = *pSet->pSmaF;
// last // sst
for (fSet.nLastF = 0; fSet.nLastF < pSet->nLastF; fSet.nLastF++) { for (fSet.nSstF = 0; fSet.nSstF < pSet->nSstF; fSet.nSstF++) {
fSet.aLastF[fSet.nLastF] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); fSet.aSstF[fSet.nSstF] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
if (fSet.aLastF[fSet.nLastF] == NULL) { if (fSet.aSstF[fSet.nSstF] == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
*fSet.aLastF[fSet.nLastF] = *pSet->aLastF[fSet.nLastF]; *fSet.aSstF[fSet.nSstF] = *pSet->aSstF[fSet.nSstF];
} }
if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) { if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) {
@ -630,28 +632,28 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) {
*pDFileSet->pHeadF = *pSet->pHeadF; *pDFileSet->pHeadF = *pSet->pHeadF;
*pDFileSet->pDataF = *pSet->pDataF; *pDFileSet->pDataF = *pSet->pDataF;
*pDFileSet->pSmaF = *pSet->pSmaF; *pDFileSet->pSmaF = *pSet->pSmaF;
// last // sst
if (pSet->nLastF > pDFileSet->nLastF) { if (pSet->nSstF > pDFileSet->nSstF) {
ASSERT(pSet->nLastF == pDFileSet->nLastF + 1); ASSERT(pSet->nSstF == pDFileSet->nSstF + 1);
pDFileSet->aLastF[pDFileSet->nLastF] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); pDFileSet->aSstF[pDFileSet->nSstF] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
if (pDFileSet->aLastF[pDFileSet->nLastF] == NULL) { if (pDFileSet->aSstF[pDFileSet->nSstF] == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
*pDFileSet->aLastF[pDFileSet->nLastF] = *pSet->aLastF[pSet->nLastF - 1]; *pDFileSet->aSstF[pDFileSet->nSstF] = *pSet->aSstF[pSet->nSstF - 1];
pDFileSet->nLastF++; pDFileSet->nSstF++;
} else if (pSet->nLastF < pDFileSet->nLastF) { } else if (pSet->nSstF < pDFileSet->nSstF) {
ASSERT(pSet->nLastF == 1); ASSERT(pSet->nSstF == 1);
for (int32_t iLast = 1; iLast < pDFileSet->nLastF; iLast++) { for (int32_t iSst = 1; iSst < pDFileSet->nSstF; iSst++) {
taosMemoryFree(pDFileSet->aLastF[iLast]); taosMemoryFree(pDFileSet->aSstF[iSst]);
} }
*pDFileSet->aLastF[0] = *pSet->aLastF[0]; *pDFileSet->aSstF[0] = *pSet->aSstF[0];
pDFileSet->nLastF = 1; pDFileSet->nSstF = 1;
} else { } else {
for (int32_t iLast = 0; iLast < pSet->nLastF; iLast++) { for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
*pDFileSet->aLastF[iLast] = *pSet->aLastF[iLast]; *pDFileSet->aSstF[iSst] = *pSet->aSstF[iSst];
} }
} }
@ -659,8 +661,8 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) {
} }
} }
ASSERT(pSet->nLastF == 1); ASSERT(pSet->nSstF == 1);
SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid, .nLastF = 1}; SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid, .nSstF = 1};
// head // head
fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile));
@ -686,13 +688,13 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) {
} }
*fSet.pSmaF = *pSet->pSmaF; *fSet.pSmaF = *pSet->pSmaF;
// last // sst
fSet.aLastF[0] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); fSet.aSstF[0] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
if (fSet.aLastF[0] == NULL) { if (fSet.aSstF[0] == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
*fSet.aLastF[0] = *pSet->aLastF[0]; *fSet.aSstF[0] = *pSet->aSstF[0];
if (taosArrayInsert(pFS->aDFileSet, idx, &fSet) == NULL) { if (taosArrayInsert(pFS->aDFileSet, idx, &fSet) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
@ -860,76 +862,76 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
pSetOld->pSmaF->size = pSetNew->pSmaF->size; pSetOld->pSmaF->size = pSetNew->pSmaF->size;
} }
// last // sst
if (sameDisk) { if (sameDisk) {
if (pSetNew->nLastF > pSetOld->nLastF) { if (pSetNew->nSstF > pSetOld->nSstF) {
ASSERT(pSetNew->nLastF = pSetOld->nLastF + 1); ASSERT(pSetNew->nSstF = pSetOld->nSstF + 1);
pSetOld->aLastF[pSetOld->nLastF] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); pSetOld->aSstF[pSetOld->nSstF] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
if (pSetOld->aLastF[pSetOld->nLastF] == NULL) { if (pSetOld->aSstF[pSetOld->nSstF] == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
*pSetOld->aLastF[pSetOld->nLastF] = *pSetNew->aLastF[pSetOld->nLastF]; *pSetOld->aSstF[pSetOld->nSstF] = *pSetNew->aSstF[pSetOld->nSstF];
pSetOld->aLastF[pSetOld->nLastF]->nRef = 1; pSetOld->aSstF[pSetOld->nSstF]->nRef = 1;
pSetOld->nLastF++; pSetOld->nSstF++;
} else if (pSetNew->nLastF < pSetOld->nLastF) { } else if (pSetNew->nSstF < pSetOld->nSstF) {
ASSERT(pSetNew->nLastF == 1); ASSERT(pSetNew->nSstF == 1);
for (int32_t iLast = 0; iLast < pSetOld->nLastF; iLast++) { for (int32_t iSst = 0; iSst < pSetOld->nSstF; iSst++) {
SLastFile *pLastFile = pSetOld->aLastF[iLast]; SSstFile *pSstFile = pSetOld->aSstF[iSst];
nRef = atomic_sub_fetch_32(&pLastFile->nRef, 1); nRef = atomic_sub_fetch_32(&pSstFile->nRef, 1);
if (nRef == 0) { if (nRef == 0) {
tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pLastFile, fname); tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSstFile, fname);
taosRemoveFile(fname); taosRemoveFile(fname);
taosMemoryFree(pLastFile); taosMemoryFree(pSstFile);
} }
pSetOld->aLastF[iLast] = NULL; pSetOld->aSstF[iSst] = NULL;
} }
pSetOld->nLastF = 1; pSetOld->nSstF = 1;
pSetOld->aLastF[0] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); pSetOld->aSstF[0] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
if (pSetOld->aLastF[0] == NULL) { if (pSetOld->aSstF[0] == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
*pSetOld->aLastF[0] = *pSetNew->aLastF[0]; *pSetOld->aSstF[0] = *pSetNew->aSstF[0];
pSetOld->aLastF[0]->nRef = 1; pSetOld->aSstF[0]->nRef = 1;
} else { } else {
for (int32_t iLast = 0; iLast < pSetOld->nLastF; iLast++) { for (int32_t iSst = 0; iSst < pSetOld->nSstF; iSst++) {
SLastFile *pLastFile = pSetOld->aLastF[iLast]; SSstFile *pSstFile = pSetOld->aSstF[iSst];
nRef = atomic_sub_fetch_32(&pLastFile->nRef, 1); nRef = atomic_sub_fetch_32(&pSstFile->nRef, 1);
if (nRef == 0) { if (nRef == 0) {
tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pLastFile, fname); tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSstFile, fname);
taosRemoveFile(fname); taosRemoveFile(fname);
taosMemoryFree(pLastFile); taosMemoryFree(pSstFile);
} }
pSetOld->aLastF[iLast] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); pSetOld->aSstF[iSst] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
if (pSetOld->aLastF[iLast] == NULL) { if (pSetOld->aSstF[iSst] == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
*pSetOld->aLastF[iLast] = *pSetNew->aLastF[iLast]; *pSetOld->aSstF[iSst] = *pSetNew->aSstF[iSst];
pSetOld->aLastF[iLast]->nRef = 1; pSetOld->aSstF[iSst]->nRef = 1;
} }
} }
} else { } else {
ASSERT(pSetOld->nLastF == pSetNew->nLastF); ASSERT(pSetOld->nSstF == pSetNew->nSstF);
for (int32_t iLast = 0; iLast < pSetOld->nLastF; iLast++) { for (int32_t iSst = 0; iSst < pSetOld->nSstF; iSst++) {
SLastFile *pLastFile = pSetOld->aLastF[iLast]; SSstFile *pSstFile = pSetOld->aSstF[iSst];
nRef = atomic_sub_fetch_32(&pLastFile->nRef, 1); nRef = atomic_sub_fetch_32(&pSstFile->nRef, 1);
if (nRef == 0) { if (nRef == 0) {
tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pLastFile, fname); tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSstFile, fname);
taosRemoveFile(fname); taosRemoveFile(fname);
taosMemoryFree(pLastFile); taosMemoryFree(pSstFile);
} }
pSetOld->aLastF[iLast] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); pSetOld->aSstF[iSst] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
if (pSetOld->aLastF[iLast] == NULL) { if (pSetOld->aSstF[iSst] == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
*pSetOld->aLastF[iLast] = *pSetNew->aLastF[iLast]; *pSetOld->aSstF[iSst] = *pSetNew->aSstF[iSst];
pSetOld->aLastF[iLast]->nRef = 1; pSetOld->aSstF[iSst]->nRef = 1;
} }
} }
@ -963,12 +965,12 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
taosMemoryFree(pSetOld->pSmaF); taosMemoryFree(pSetOld->pSmaF);
} }
for (int8_t iLast = 0; iLast < pSetOld->nLastF; iLast++) { for (int8_t iSst = 0; iSst < pSetOld->nSstF; iSst++) {
nRef = atomic_sub_fetch_32(&pSetOld->aLastF[iLast]->nRef, 1); nRef = atomic_sub_fetch_32(&pSetOld->aSstF[iSst]->nRef, 1);
if (nRef == 0) { if (nRef == 0) {
tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->aLastF[iLast], fname); tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->aSstF[iSst], fname);
taosRemoveFile(fname); taosRemoveFile(fname);
taosMemoryFree(pSetOld->aLastF[iLast]); taosMemoryFree(pSetOld->aSstF[iSst]);
} }
} }
@ -976,7 +978,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
continue; continue;
_add_new: _add_new:
fSet = (SDFileSet){.diskId = pSetNew->diskId, .fid = pSetNew->fid, .nLastF = 1}; fSet = (SDFileSet){.diskId = pSetNew->diskId, .fid = pSetNew->fid, .nSstF = 1};
// head // head
fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile));
@ -1005,15 +1007,15 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
*fSet.pSmaF = *pSetNew->pSmaF; *fSet.pSmaF = *pSetNew->pSmaF;
fSet.pSmaF->nRef = 1; fSet.pSmaF->nRef = 1;
// last // sst
ASSERT(pSetNew->nLastF == 1); ASSERT(pSetNew->nSstF == 1);
fSet.aLastF[0] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); fSet.aSstF[0] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
if (fSet.aLastF[0] == NULL) { if (fSet.aSstF[0] == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
*fSet.aLastF[0] = *pSetNew->aLastF[0]; *fSet.aSstF[0] = *pSetNew->aSstF[0];
fSet.aLastF[0]->nRef = 1; fSet.aSstF[0]->nRef = 1;
if (taosArrayInsert(pTsdb->fs.aDFileSet, iOld, &fSet) == NULL) { if (taosArrayInsert(pTsdb->fs.aDFileSet, iOld, &fSet) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
@ -1061,8 +1063,8 @@ int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS) {
nRef = atomic_fetch_add_32(&pSet->pSmaF->nRef, 1); nRef = atomic_fetch_add_32(&pSet->pSmaF->nRef, 1);
ASSERT(nRef > 0); ASSERT(nRef > 0);
for (int32_t iLast = 0; iLast < pSet->nLastF; iLast++) { for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
nRef = atomic_fetch_add_32(&pSet->aLastF[iLast]->nRef, 1); nRef = atomic_fetch_add_32(&pSet->aSstF[iSst]->nRef, 1);
ASSERT(nRef > 0); ASSERT(nRef > 0);
} }
@ -1120,14 +1122,14 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) {
taosMemoryFree(pSet->pSmaF); taosMemoryFree(pSet->pSmaF);
} }
// last // sst
for (int32_t iLast = 0; iLast < pSet->nLastF; iLast++) { for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
nRef = atomic_sub_fetch_32(&pSet->aLastF[iLast]->nRef, 1); nRef = atomic_sub_fetch_32(&pSet->aSstF[iSst]->nRef, 1);
ASSERT(nRef >= 0); ASSERT(nRef >= 0);
if (nRef == 0) { if (nRef == 0) {
tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aLastF[iLast], fname); tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[iSst], fname);
taosRemoveFile(fname); taosRemoveFile(fname);
taosMemoryFree(pSet->aLastF[iLast]); taosMemoryFree(pSet->aSstF[iSst]);
/* code */ /* code */
} }
} }

42
source/dnode/vnode/src/tsdb/tsdbFile.c
View File

@ -53,22 +53,22 @@ static int32_t tGetDataFile(uint8_t *p, SDataFile *pDataFile) {
return n; return n;
} }
int32_t tPutLastFile(uint8_t *p, SLastFile *pLastFile) { int32_t tPutSstFile(uint8_t *p, SSstFile *pSstFile) {
int32_t n = 0; int32_t n = 0;
n += tPutI64v(p ? p + n : p, pLastFile->commitID); n += tPutI64v(p ? p + n : p, pSstFile->commitID);
n += tPutI64v(p ? p + n : p, pLastFile->size); n += tPutI64v(p ? p + n : p, pSstFile->size);
n += tPutI64v(p ? p + n : p, pLastFile->offset); n += tPutI64v(p ? p + n : p, pSstFile->offset);
return n; return n;
} }
static int32_t tGetLastFile(uint8_t *p, SLastFile *pLastFile) { static int32_t tGetSstFile(uint8_t *p, SSstFile *pSstFile) {
int32_t n = 0; int32_t n = 0;
n += tGetI64v(p + n, &pLastFile->commitID); n += tGetI64v(p + n, &pSstFile->commitID);
n += tGetI64v(p + n, &pLastFile->size); n += tGetI64v(p + n, &pSstFile->size);
n += tGetI64v(p + n, &pLastFile->offset); n += tGetI64v(p + n, &pSstFile->offset);
return n; return n;
} }
@ -102,9 +102,9 @@ void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF,
TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pDataF->commitID, ".data"); TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pDataF->commitID, ".data");
} }
void tsdbLastFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SLastFile *pLastF, char fname[]) { void tsdbSstFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSstFile *pSstF, char fname[]) {
snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%df%dver%" PRId64 "%s", tfsGetDiskPath(pTsdb->pVnode->pTfs, did), snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%df%dver%" PRId64 "%s", tfsGetDiskPath(pTsdb->pVnode->pTfs, did),
TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pLastF->commitID, ".last"); TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pSstF->commitID, ".sst");
} }
void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]) { void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]) {
@ -194,10 +194,10 @@ int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) {
n += tPutDataFile(p ? p + n : p, pSet->pDataF); n += tPutDataFile(p ? p + n : p, pSet->pDataF);
n += tPutSmaFile(p ? p + n : p, pSet->pSmaF); n += tPutSmaFile(p ? p + n : p, pSet->pSmaF);
// last // sst
n += tPutU8(p ? p + n : p, pSet->nLastF); n += tPutU8(p ? p + n : p, pSet->nSstF);
for (int32_t iLast = 0; iLast < pSet->nLastF; iLast++) { for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
n += tPutLastFile(p ? p + n : p, pSet->aLastF[iLast]); n += tPutSstFile(p ? p + n : p, pSet->aSstF[iSst]);
} }
return n; return n;
@ -234,15 +234,15 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) {
pSet->pSmaF->nRef = 1; pSet->pSmaF->nRef = 1;
n += tGetSmaFile(p + n, pSet->pSmaF); n += tGetSmaFile(p + n, pSet->pSmaF);
// last // sst
n += tGetU8(p + n, &pSet->nLastF); n += tGetU8(p + n, &pSet->nSstF);
for (int32_t iLast = 0; iLast < pSet->nLastF; iLast++) { for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
pSet->aLastF[iLast] = (SLastFile *)taosMemoryCalloc(1, sizeof(SLastFile)); pSet->aSstF[iSst] = (SSstFile *)taosMemoryCalloc(1, sizeof(SSstFile));
if (pSet->aLastF[iLast] == NULL) { if (pSet->aSstF[iSst] == NULL) {
return -1; return -1;
} }
pSet->aLastF[iLast]->nRef = 1; pSet->aSstF[iSst]->nRef = 1;
n += tGetLastFile(p + n, pSet->aLastF[iLast]); n += tGetSstFile(p + n, pSet->aSstF[iSst]);
} }
return n; return n;

100
source/dnode/vnode/src/tsdb/tsdbMergeTree.c
View File

@ -18,12 +18,12 @@
// SLDataIter ================================================= // SLDataIter =================================================
typedef struct SLDataIter { typedef struct SLDataIter {
SRBTreeNode node; SRBTreeNode node;
SBlockL *pBlockL; SSstBlk *pSstBlk;
SDataFReader *pReader; SDataFReader *pReader;
int32_t iLast; int32_t iSst;
int8_t backward; int8_t backward;
SArray *aBlockL; SArray *aSstBlk;
int32_t iBlockL; int32_t iSstBlk;
SBlockData bData; SBlockData bData;
int32_t iRow; int32_t iRow;
SRowInfo rInfo; SRowInfo rInfo;
@ -32,7 +32,7 @@ typedef struct SLDataIter {
SVersionRange verRange; SVersionRange verRange;
} SLDataIter; } SLDataIter;
int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iLast, int8_t backward, uint64_t uid, int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iSst, int8_t backward, uint64_t uid,
STimeWindow *pTimeWindow, SVersionRange *pRange) { STimeWindow *pTimeWindow, SVersionRange *pRange) {
int32_t code = 0; int32_t code = 0;
*pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); *pIter = taosMemoryCalloc(1, sizeof(SLDataIter));
@ -45,10 +45,10 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
(*pIter)->timeWindow = *pTimeWindow; (*pIter)->timeWindow = *pTimeWindow;
(*pIter)->verRange = *pRange; (*pIter)->verRange = *pRange;
(*pIter)->pReader = pReader; (*pIter)->pReader = pReader;
(*pIter)->iLast = iLast; (*pIter)->iSst = iSst;
(*pIter)->backward = backward; (*pIter)->backward = backward;
(*pIter)->aBlockL = taosArrayInit(0, sizeof(SBlockL)); (*pIter)->aSstBlk = taosArrayInit(0, sizeof(SSstBlk));
if ((*pIter)->aBlockL == NULL) { if ((*pIter)->aSstBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
@ -58,18 +58,18 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
goto _exit; goto _exit;
} }
code = tsdbReadBlockL(pReader, iLast, (*pIter)->aBlockL); code = tsdbReadSstBlk(pReader, iSst, (*pIter)->aSstBlk);
if (code) { if (code) {
goto _exit; goto _exit;
} }
size_t size = taosArrayGetSize((*pIter)->aBlockL); size_t size = taosArrayGetSize((*pIter)->aSstBlk);
// find the start block // find the start block
int32_t index = -1; int32_t index = -1;
if (!backward) { // asc if (!backward) { // asc
for (int32_t i = 0; i < size; ++i) { for (int32_t i = 0; i < size; ++i) {
SBlockL *p = taosArrayGet((*pIter)->aBlockL, i); SSstBlk *p = taosArrayGet((*pIter)->aSstBlk, i);
if (p->minUid <= uid && p->maxUid >= uid) { if (p->minUid <= uid && p->maxUid >= uid) {
index = i; index = i;
break; break;
@ -77,7 +77,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
} }
} else { // desc } else { // desc
for (int32_t i = size - 1; i >= 0; --i) { for (int32_t i = size - 1; i >= 0; --i) {
SBlockL *p = taosArrayGet((*pIter)->aBlockL, i); SSstBlk *p = taosArrayGet((*pIter)->aSstBlk, i);
if (p->minUid <= uid && p->maxUid >= uid) { if (p->minUid <= uid && p->maxUid >= uid) {
index = i; index = i;
break; break;
@ -85,9 +85,9 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
} }
} }
(*pIter)->iBlockL = index; (*pIter)->iSstBlk = index;
if (index != -1) { if (index != -1) {
(*pIter)->pBlockL = taosArrayGet((*pIter)->aBlockL, (*pIter)->iBlockL); (*pIter)->pSstBlk = taosArrayGet((*pIter)->aSstBlk, (*pIter)->iSstBlk);
} }
_exit: _exit:
@ -96,20 +96,20 @@ _exit:
void tLDataIterClose(SLDataIter *pIter) { void tLDataIterClose(SLDataIter *pIter) {
tBlockDataDestroy(&pIter->bData, 1); tBlockDataDestroy(&pIter->bData, 1);
taosArrayDestroy(pIter->aBlockL); taosArrayDestroy(pIter->aSstBlk);
taosMemoryFree(pIter); taosMemoryFree(pIter);
} }
extern int32_t tsdbReadLastBlockEx(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, SBlockData *pBlockData); extern int32_t tsdbReadSstBlockEx(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData);
void tLDataIterNextBlock(SLDataIter *pIter) { void tLDataIterNextBlock(SLDataIter *pIter) {
int32_t step = pIter->backward? -1:1; int32_t step = pIter->backward ? -1 : 1;
pIter->iBlockL += step; pIter->iSstBlk += step;
int32_t index = -1; int32_t index = -1;
size_t size = taosArrayGetSize(pIter->aBlockL); size_t size = taosArrayGetSize(pIter->aSstBlk);
for(int32_t i = pIter->iBlockL; i < size && i >= 0; i += step) { for (int32_t i = pIter->iSstBlk; i < size && i >= 0; i += step) {
SBlockL *p = taosArrayGet(pIter->aBlockL, i); SSstBlk *p = taosArrayGet(pIter->aSstBlk, i);
if ((!pIter->backward) && p->minUid > pIter->uid) { if ((!pIter->backward) && p->minUid > pIter->uid) {
break; break;
} }
@ -125,16 +125,16 @@ void tLDataIterNextBlock(SLDataIter *pIter) {
} }
if (index == -1) { if (index == -1) {
pIter->pBlockL = NULL; pIter->pSstBlk = NULL;
} else { } else {
pIter->pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, pIter->iBlockL); pIter->pSstBlk = (SSstBlk *)taosArrayGet(pIter->aSstBlk, pIter->iSstBlk);
} }
} }
static void findNextValidRow(SLDataIter* pIter) { static void findNextValidRow(SLDataIter *pIter) {
int32_t step = pIter->backward? -1:1; int32_t step = pIter->backward ? -1 : 1;
bool hasVal = false; bool hasVal = false;
int32_t i = pIter->iRow; int32_t i = pIter->iRow;
for (; i < pIter->bData.nRow && i >= 0; i += step) { for (; i < pIter->bData.nRow && i >= 0; i += step) {
if (pIter->bData.aUid != NULL) { if (pIter->bData.aUid != NULL) {
@ -154,7 +154,7 @@ static void findNextValidRow(SLDataIter* pIter) {
} }
int64_t ts = pIter->bData.aTSKEY[i]; int64_t ts = pIter->bData.aTSKEY[i];
if (!pIter->backward) { // asc if (!pIter->backward) { // asc
if (ts > pIter->timeWindow.ekey) { // no more data if (ts > pIter->timeWindow.ekey) { // no more data
break; break;
} else if (ts < pIter->timeWindow.skey) { } else if (ts < pIter->timeWindow.skey) {
@ -190,45 +190,45 @@ static void findNextValidRow(SLDataIter* pIter) {
break; break;
} }
pIter->iRow = (hasVal)? i:-1; pIter->iRow = (hasVal) ? i : -1;
} }
bool tLDataIterNextRow(SLDataIter *pIter) { bool tLDataIterNextRow(SLDataIter *pIter) {
int32_t code = 0; int32_t code = 0;
int32_t step = pIter->backward? -1:1; int32_t step = pIter->backward ? -1 : 1;
// no qualified last file block in current file, no need to fetch row // no qualified last file block in current file, no need to fetch row
if (pIter->pBlockL == NULL) { if (pIter->pSstBlk == NULL) {
return false; return false;
} }
int32_t iBlockL = pIter->iBlockL; int32_t iBlockL = pIter->iSstBlk;
if (pIter->bData.nRow == 0 && pIter->pBlockL != NULL) { // current block not loaded yet if (pIter->bData.nRow == 0 && pIter->pSstBlk != NULL) { // current block not loaded yet
code = tsdbReadLastBlockEx(pIter->pReader, pIter->iLast, pIter->pBlockL, &pIter->bData); code = tsdbReadSstBlockEx(pIter->pReader, pIter->iSst, pIter->pSstBlk, &pIter->bData);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _exit; goto _exit;
} }
pIter->iRow = (pIter->backward)? pIter->bData.nRow:-1; pIter->iRow = (pIter->backward) ? pIter->bData.nRow : -1;
} }
pIter->iRow += step; pIter->iRow += step;
while(1) { while (1) {
findNextValidRow(pIter); findNextValidRow(pIter);
if (pIter->iRow >= pIter->bData.nRow || pIter->iRow < 0) { if (pIter->iRow >= pIter->bData.nRow || pIter->iRow < 0) {
tLDataIterNextBlock(pIter); tLDataIterNextBlock(pIter);
if (pIter->pBlockL == NULL) { // no more data if (pIter->pSstBlk == NULL) { // no more data
goto _exit; goto _exit;
} }
} else { } else {
break; break;
} }
if (iBlockL != pIter->iBlockL) { if (iBlockL != pIter->iSstBlk) {
code = tsdbReadLastBlockEx(pIter->pReader, pIter->iLast, pIter->pBlockL, &pIter->bData); code = tsdbReadSstBlockEx(pIter->pReader, pIter->iSst, pIter->pSstBlk, &pIter->bData);
if (code) { if (code) {
goto _exit; goto _exit;
} }
@ -241,16 +241,14 @@ bool tLDataIterNextRow(SLDataIter *pIter) {
pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow);
_exit: _exit:
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
terrno = code; terrno = code;
} }
return (code == TSDB_CODE_SUCCESS) && (pIter->pBlockL != NULL); return (code == TSDB_CODE_SUCCESS) && (pIter->pSstBlk != NULL);
} }
SRowInfo *tLDataIterGet(SLDataIter *pIter) { SRowInfo *tLDataIterGet(SLDataIter *pIter) { return &pIter->rInfo; }
return &pIter->rInfo;
}
// SMergeTree ================================================= // SMergeTree =================================================
static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) { static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) {
@ -287,7 +285,7 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFRead
int32_t code = TSDB_CODE_OUT_OF_MEMORY; int32_t code = TSDB_CODE_OUT_OF_MEMORY;
struct SLDataIter* pIterList[TSDB_DEFAULT_LAST_FILE] = {0}; struct SLDataIter* pIterList[TSDB_DEFAULT_LAST_FILE] = {0};
for(int32_t i = 0; i < pFReader->pSet->nLastF; ++i) { // open all last file for(int32_t i = 0; i < pFReader->pSet->nSstF; ++i) { // open all last file
code = tLDataIterOpen(&pIterList[i], pFReader, i, pMTree->backward, uid, pTimeWindow, pVerRange); code = tLDataIterOpen(&pIterList[i], pFReader, i, pMTree->backward, uid, pTimeWindow, pVerRange);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _end; goto _end;
@ -311,7 +309,7 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFRead
void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter) { tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pIter); } void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter) { tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pIter); }
bool tMergeTreeNext(SMergeTree* pMTree) { bool tMergeTreeNext(SMergeTree *pMTree) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (pMTree->pIter) { if (pMTree->pIter) {
SLDataIter *pIter = pMTree->pIter; SLDataIter *pIter = pMTree->pIter;
@ -344,14 +342,12 @@ bool tMergeTreeNext(SMergeTree* pMTree) {
return pMTree->pIter != NULL; return pMTree->pIter != NULL;
} }
TSDBROW tMergeTreeGetRow(SMergeTree* pMTree) { TSDBROW tMergeTreeGetRow(SMergeTree *pMTree) { return pMTree->pIter->rInfo.row; }
return pMTree->pIter->rInfo.row;
}
void tMergeTreeClose(SMergeTree* pMTree) { void tMergeTreeClose(SMergeTree *pMTree) {
size_t size = taosArrayGetSize(pMTree->pIterList); size_t size = taosArrayGetSize(pMTree->pIterList);
for(int32_t i = 0; i < size; ++i) { for (int32_t i = 0; i < size; ++i) {
SLDataIter* pIter = taosArrayGetP(pMTree->pIterList, i); SLDataIter *pIter = taosArrayGetP(pMTree->pIterList, i);
tLDataIterClose(pIter); tLDataIterClose(pIter);
} }

104
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -109,7 +109,7 @@ typedef struct SDataBlockIter {
int32_t index; int32_t index;
SArray* blockList; // SArray<SFileDataBlockInfo> SArray* blockList; // SArray<SFileDataBlockInfo>
int32_t order; int32_t order;
SBlock block; // current SBlock data SDataBlk block; // current SDataBlk data
SHashObj* pTableMap; SHashObj* pTableMap;
} SDataBlockIter; } SDataBlockIter;
@ -176,10 +176,10 @@ static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pR
static void setComposedBlockFlag(STsdbReader* pReader, bool composed); static void setComposedBlockFlag(STsdbReader* pReader, bool composed);
static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order); static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order);
static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList,
STsdbReader* pReader, bool* freeTSRow); STSRow** pTSRow, STsdbReader* pReader, bool* freeTSRow);
static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo,
STSRow** pTSRow); STsdbReader* pReader, STSRow** pTSRow);
static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key,
STsdbReader* pReader); STsdbReader* pReader);
@ -590,8 +590,8 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN
sizeInDisk += pScanInfo->mapData.nData; sizeInDisk += pScanInfo->mapData.nData;
for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) { for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) {
SBlock block = {0}; SDataBlk block = {0};
tMapDataGetItemByIdx(&pScanInfo->mapData, j, &block, tGetBlock); tMapDataGetItemByIdx(&pScanInfo->mapData, j, &block, tGetDataBlk);
// 1. time range check // 1. time range check
if (block.minKey.ts > pReader->window.ekey || block.maxKey.ts < pReader->window.skey) { if (block.minKey.ts > pReader->window.ekey || block.maxKey.ts < pReader->window.skey) {
@ -617,7 +617,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN
} }
} }
pBlockNum->numOfLastFiles = pReader->pFileReader->pSet->nLastF; pBlockNum->numOfLastFiles = pReader->pFileReader->pSet->nSstF;
int32_t total = pBlockNum->numOfLastFiles + pBlockNum->numOfBlocks; int32_t total = pBlockNum->numOfLastFiles + pBlockNum->numOfBlocks;
double el = (taosGetTimestampUs() - st) / 1000.0; double el = (taosGetTimestampUs() - st) / 1000.0;
@ -665,7 +665,7 @@ static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) {
return pBlockInfo; return pBlockInfo;
} }
static SBlock* getCurrentBlock(SDataBlockIter* pBlockIter) { return &pBlockIter->block; } static SDataBlk* getCurrentBlock(SDataBlockIter* pBlockIter) { return &pBlockIter->block; }
static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) { static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) {
SReaderStatus* pStatus = &pReader->status; SReaderStatus* pStatus = &pReader->status;
@ -673,7 +673,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
SBlockData* pBlockData = &pStatus->fileBlockData; SBlockData* pBlockData = &pStatus->fileBlockData;
SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter);
SBlock* pBlock = getCurrentBlock(pBlockIter); SDataBlk* pBlock = getCurrentBlock(pBlockIter);
SSDataBlock* pResBlock = pReader->pResBlock; SSDataBlock* pResBlock = pReader->pResBlock;
int32_t numOfOutputCols = blockDataGetNumOfCols(pResBlock); int32_t numOfOutputCols = blockDataGetNumOfCols(pResBlock);
@ -758,8 +758,8 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
ASSERT(pBlockInfo != NULL); ASSERT(pBlockInfo != NULL);
SBlock* pBlock = getCurrentBlock(pBlockIter); SDataBlk* pBlock = getCurrentBlock(pBlockIter);
int32_t code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData); int32_t code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64
", rows:%d, code:%s %s", ", rows:%d, code:%s %s",
@ -836,7 +836,7 @@ static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter) {
if (pBlockInfo != NULL) { if (pBlockInfo != NULL) {
STableBlockScanInfo* pScanInfo = taosHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); STableBlockScanInfo* pScanInfo = taosHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid));
int32_t* mapDataIndex = taosArrayGet(pScanInfo->pBlockList, pBlockInfo->tbBlockIdx); int32_t* mapDataIndex = taosArrayGet(pScanInfo->pBlockList, pBlockInfo->tbBlockIdx);
tMapDataGetItemByIdx(&pScanInfo->mapData, *mapDataIndex, &pBlockIter->block, tGetBlock); tMapDataGetItemByIdx(&pScanInfo->mapData, *mapDataIndex, &pBlockIter->block, tGetDataBlk);
} }
#if 0 #if 0
@ -887,12 +887,12 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte
} }
sup.pDataBlockInfo[sup.numOfTables] = (SBlockOrderWrapper*)buf; sup.pDataBlockInfo[sup.numOfTables] = (SBlockOrderWrapper*)buf;
SBlock block = {0}; SDataBlk block = {0};
for (int32_t k = 0; k < num; ++k) { for (int32_t k = 0; k < num; ++k) {
SBlockOrderWrapper wrapper = {0}; SBlockOrderWrapper wrapper = {0};
int32_t* mapDataIndex = taosArrayGet(pTableScanInfo->pBlockList, k); int32_t* mapDataIndex = taosArrayGet(pTableScanInfo->pBlockList, k);
tMapDataGetItemByIdx(&pTableScanInfo->mapData, *mapDataIndex, &block, tGetBlock); tMapDataGetItemByIdx(&pTableScanInfo->mapData, *mapDataIndex, &block, tGetDataBlk);
wrapper.uid = pTableScanInfo->uid; wrapper.uid = pTableScanInfo->uid;
wrapper.offset = block.aSubBlock[0].offset; wrapper.offset = block.aSubBlock[0].offset;
@ -981,15 +981,15 @@ static bool blockIteratorNext(SDataBlockIter* pBlockIter) {
/** /**
* This is an two rectangles overlap cases. * This is an two rectangles overlap cases.
*/ */
static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* pVerRange, SBlock* pBlock) { static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* pVerRange, SDataBlk* pBlock) {
return (pWindow->ekey < pBlock->maxKey.ts && pWindow->ekey >= pBlock->minKey.ts) || return (pWindow->ekey < pBlock->maxKey.ts && pWindow->ekey >= pBlock->minKey.ts) ||
(pWindow->skey > pBlock->minKey.ts && pWindow->skey <= pBlock->maxKey.ts) || (pWindow->skey > pBlock->minKey.ts && pWindow->skey <= pBlock->maxKey.ts) ||
(pVerRange->minVer > pBlock->minVer && pVerRange->minVer <= pBlock->maxVer) || (pVerRange->minVer > pBlock->minVer && pVerRange->minVer <= pBlock->maxVer) ||
(pVerRange->maxVer < pBlock->maxVer && pVerRange->maxVer >= pBlock->minVer); (pVerRange->maxVer < pBlock->maxVer && pVerRange->maxVer >= pBlock->minVer);
} }
static SBlock* getNeighborBlockOfSameTable(SFileDataBlockInfo* pFBlockInfo, STableBlockScanInfo* pTableBlockScanInfo, static SDataBlk* getNeighborBlockOfSameTable(SFileDataBlockInfo* pFBlockInfo, STableBlockScanInfo* pTableBlockScanInfo,
int32_t* nextIndex, int32_t order) { int32_t* nextIndex, int32_t order) {
bool asc = ASCENDING_TRAVERSE(order); bool asc = ASCENDING_TRAVERSE(order);
if (asc && pFBlockInfo->tbBlockIdx >= taosArrayGetSize(pTableBlockScanInfo->pBlockList) - 1) { if (asc && pFBlockInfo->tbBlockIdx >= taosArrayGetSize(pTableBlockScanInfo->pBlockList) - 1) {
return NULL; return NULL;
@ -1002,10 +1002,10 @@ static SBlock* getNeighborBlockOfSameTable(SFileDataBlockInfo* pFBlockInfo, STab
int32_t step = asc ? 1 : -1; int32_t step = asc ? 1 : -1;
*nextIndex = pFBlockInfo->tbBlockIdx + step; *nextIndex = pFBlockInfo->tbBlockIdx + step;
SBlock* pBlock = taosMemoryCalloc(1, sizeof(SBlock)); SDataBlk* pBlock = taosMemoryCalloc(1, sizeof(SDataBlk));
int32_t* indexInMapdata = taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex); int32_t* indexInMapdata = taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex);
tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, *indexInMapdata, pBlock, tGetBlock); tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, *indexInMapdata, pBlock, tGetDataBlk);
return pBlock; return pBlock;
} }
@ -1048,7 +1048,7 @@ static int32_t setFileBlockActiveInBlockIter(SDataBlockIter* pBlockIter, int32_t
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static bool overlapWithNeighborBlock(SBlock* pBlock, SBlock* pNeighbor, int32_t order) { static bool overlapWithNeighborBlock(SDataBlk* pBlock, SDataBlk* pNeighbor, int32_t order) {
// it is the last block in current file, no chance to overlap with neighbor blocks. // it is the last block in current file, no chance to overlap with neighbor blocks.
if (ASCENDING_TRAVERSE(order)) { if (ASCENDING_TRAVERSE(order)) {
return pBlock->maxKey.ts == pNeighbor->minKey.ts; return pBlock->maxKey.ts == pNeighbor->minKey.ts;
@ -1057,19 +1057,19 @@ static bool overlapWithNeighborBlock(SBlock* pBlock, SBlock* pNeighbor, int32_t
} }
} }
static bool bufferDataInFileBlockGap(int32_t order, TSDBKEY key, SBlock* pBlock) { static bool bufferDataInFileBlockGap(int32_t order, TSDBKEY key, SDataBlk* pBlock) {
bool ascScan = ASCENDING_TRAVERSE(order); bool ascScan = ASCENDING_TRAVERSE(order);
return (ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts <= pBlock->minKey.ts)) || return (ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts <= pBlock->minKey.ts)) ||
(!ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts >= pBlock->maxKey.ts)); (!ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts >= pBlock->maxKey.ts));
} }
static bool keyOverlapFileBlock(TSDBKEY key, SBlock* pBlock, SVersionRange* pVerRange) { static bool keyOverlapFileBlock(TSDBKEY key, SDataBlk* pBlock, SVersionRange* pVerRange) {
return (key.ts >= pBlock->minKey.ts && key.ts <= pBlock->maxKey.ts) && (pBlock->maxVer >= pVerRange->minVer) && return (key.ts >= pBlock->minKey.ts && key.ts <= pBlock->maxKey.ts) && (pBlock->maxVer >= pVerRange->minVer) &&
(pBlock->minVer <= pVerRange->maxVer); (pBlock->minVer <= pVerRange->maxVer);
} }
static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock) { static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SDataBlk* pBlock) {
size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline); size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline);
for (int32_t i = pBlockScanInfo->fileDelIndex; i < num; i += 1) { for (int32_t i = pBlockScanInfo->fileDelIndex; i < num; i += 1) {
@ -1103,7 +1103,7 @@ static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, cons
return false; return false;
} }
static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock, int32_t order) { static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SDataBlk* pBlock, int32_t order) {
if (pBlockScanInfo->delSkyline == NULL) { if (pBlockScanInfo->delSkyline == NULL) {
return false; return false;
} }
@ -1138,10 +1138,10 @@ static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBl
// 3. current timestamp should not be overlap with each other // 3. current timestamp should not be overlap with each other
// 4. output buffer should be large enough to hold all rows in current block // 4. output buffer should be large enough to hold all rows in current block
// 5. delete info should not overlap with current block data // 5. delete info should not overlap with current block data
static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBlock, SBlock* pBlock, static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBlock, SDataBlk* pBlock,
STableBlockScanInfo* pScanInfo, TSDBKEY key, SLastBlockReader* pLastBlockReader) { STableBlockScanInfo* pScanInfo, TSDBKEY key, SLastBlockReader* pLastBlockReader) {
int32_t neighborIndex = 0; int32_t neighborIndex = 0;
SBlock* pNeighbor = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &neighborIndex, pReader->order); SDataBlk* pNeighbor = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &neighborIndex, pReader->order);
// overlap with neighbor // overlap with neighbor
bool overlapWithNeighbor = false; bool overlapWithNeighbor = false;
@ -1158,9 +1158,9 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBloc
// todo // todo
bool overlapWithlastBlock = false; bool overlapWithlastBlock = false;
#if 0 #if 0
if (taosArrayGetSize(pLastBlockReader->pBlockL) > 0 && (pLastBlockReader->currentBlockIndex != -1)) { if (taosArrayGetSize(pLastBlockReader->pSstBlk) > 0 && (pLastBlockReader->currentBlockIndex != -1)) {
SBlockL* pBlockL = taosArrayGet(pLastBlockReader->pBlockL, pLastBlockReader->currentBlockIndex); SSstBlk* pSstBlk = taosArrayGet(pLastBlockReader->pSstBlk, pLastBlockReader->currentBlockIndex);
overlapWithlastBlock = !(pBlock->maxKey.ts < pBlockL->minKey || pBlock->minKey.ts > pBlockL->maxKey); overlapWithlastBlock = !(pBlock->maxKey.ts < pSstBlk->minKey || pBlock->minKey.ts > pSstBlk->maxKey);
} }
#endif #endif
@ -1374,7 +1374,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, STsdbReader* pReader, static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, STsdbReader* pReader,
STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData,
bool mergeBlockData) { bool mergeBlockData) {
int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader);
STSRow* pTSRow = NULL; STSRow* pTSRow = NULL;
SRowMerger merge = {0}; SRowMerger merge = {0};
@ -1942,9 +1942,7 @@ static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) {
return key.ts; return key.ts;
} }
static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { return pLastBlockReader->mergeTree.pIter != NULL; }
return pLastBlockReader->mergeTree.pIter != NULL;
}
int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key,
STsdbReader* pReader) { STsdbReader* pReader) {
@ -2032,7 +2030,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
pDumpInfo->rowIndex += step; pDumpInfo->rowIndex += step;
SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter);
if (pDumpInfo->rowIndex >= pBlock->nRow || pDumpInfo->rowIndex < 0) { if (pDumpInfo->rowIndex >= pBlock->nRow || pDumpInfo->rowIndex < 0) {
setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order);
break; break;
@ -2040,7 +2038,6 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
} }
} }
bool hasBlockLData = hasDataInLastBlock(pLastBlockReader); bool hasBlockLData = hasDataInLastBlock(pLastBlockReader);
// no data in last block and block, no need to proceed. // no data in last block and block, no need to proceed.
@ -2052,7 +2049,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
// currently loaded file data block is consumed // currently loaded file data block is consumed
if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) {
SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter);
setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order);
break; break;
} }
@ -2203,7 +2200,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) {
return code; return code;
} }
if (taosArrayGetSize(pIndexList) > 0 || pReader->pFileReader->pSet->nLastF > 0) { if (taosArrayGetSize(pIndexList) > 0 || pReader->pFileReader->pSet->nSstF > 0) {
code = doLoadFileBlock(pReader, pIndexList, pBlockNum); code = doLoadFileBlock(pReader, pIndexList, pBlockNum);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
taosArrayDestroy(pIndexList); taosArrayDestroy(pIndexList);
@ -2341,8 +2338,8 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) {
} }
static int32_t doBuildDataBlock(STsdbReader* pReader) { static int32_t doBuildDataBlock(STsdbReader* pReader) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SBlock* pBlock = NULL; SDataBlk* pBlock = NULL;
SReaderStatus* pStatus = &pReader->status; SReaderStatus* pStatus = &pReader->status;
SDataBlockIter* pBlockIter = &pStatus->blockIter; SDataBlockIter* pBlockIter = &pStatus->blockIter;
@ -2439,7 +2436,7 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) {
// set the correct start position in case of the first/last file block, according to the query time window // set the correct start position in case of the first/last file block, according to the query time window
static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) {
SBlock* pBlock = getCurrentBlock(pBlockIter); SDataBlk* pBlock = getCurrentBlock(pBlockIter);
SReaderStatus* pStatus = &pReader->status; SReaderStatus* pStatus = &pReader->status;
@ -2810,7 +2807,7 @@ typedef enum {
CHECK_FILEBLOCK_QUIT = 0x2, CHECK_FILEBLOCK_QUIT = 0x2,
} CHECK_FILEBLOCK_STATE; } CHECK_FILEBLOCK_STATE;
static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanInfo* pScanInfo, SBlock* pBlock, static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanInfo* pScanInfo, SDataBlk* pBlock,
SFileDataBlockInfo* pFBlock, SRowMerger* pMerger, int64_t key, SFileDataBlockInfo* pFBlock, SRowMerger* pMerger, int64_t key,
CHECK_FILEBLOCK_STATE* state) { CHECK_FILEBLOCK_STATE* state) {
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
@ -2819,8 +2816,8 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn
*state = CHECK_FILEBLOCK_QUIT; *state = CHECK_FILEBLOCK_QUIT;
int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1;
int32_t nextIndex = -1; int32_t nextIndex = -1;
SBlock* pNeighborBlock = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &nextIndex, pReader->order); SDataBlk* pNeighborBlock = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &nextIndex, pReader->order);
if (pNeighborBlock == NULL) { // do nothing if (pNeighborBlock == NULL) { // do nothing
return 0; return 0;
} }
@ -2884,7 +2881,7 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc
CHECK_FILEBLOCK_STATE st; CHECK_FILEBLOCK_STATE st;
SFileDataBlockInfo* pFileBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); SFileDataBlockInfo* pFileBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter);
SBlock* pCurrentBlock = getCurrentBlock(&pReader->status.blockIter); SDataBlk* pCurrentBlock = getCurrentBlock(&pReader->status.blockIter);
checkForNeighborFileBlock(pReader, pScanInfo, pCurrentBlock, pFileBlockInfo, pMerger, key, &st); checkForNeighborFileBlock(pReader, pScanInfo, pCurrentBlock, pFileBlockInfo, pMerger, key, &st);
if (st == CHECK_FILEBLOCK_QUIT) { if (st == CHECK_FILEBLOCK_QUIT) {
break; break;
@ -2964,7 +2961,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter,
} }
int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader,
STSRow** pTSRow) { STSRow** pTSRow) {
SRowMerger merge = {0}; SRowMerger merge = {0};
TSDBKEY k = TSDBROW_KEY(pRow); TSDBKEY k = TSDBROW_KEY(pRow);
@ -3038,7 +3035,8 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR
} }
if (pBlockScanInfo->iter.hasVal && pRow != NULL) { if (pBlockScanInfo->iter.hasVal && pRow != NULL) {
return doMergeMemTableMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader, freeTSRow); return doMergeMemTableMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader,
freeTSRow);
} }
if (pBlockScanInfo->iiter.hasVal && piRow != NULL) { if (pBlockScanInfo->iiter.hasVal && piRow != NULL) {
@ -3467,12 +3465,12 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS
SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter); SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter);
SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter);
int64_t stime = taosGetTimestampUs(); int64_t stime = taosGetTimestampUs();
SBlockLoadSuppInfo* pSup = &pReader->suppInfo; SBlockLoadSuppInfo* pSup = &pReader->suppInfo;
if (tBlockHasSma(pBlock)) { if (tDataBlkHasSma(pBlock)) {
code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pSup->pColAgg); code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pSup->pColAgg);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64 ", code:%s, %s", 0, pFBlock->uid, tstrerror(code), tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64 ", code:%s, %s", 0, pFBlock->uid, tstrerror(code),
@ -3649,7 +3647,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa
while (true) { while (true) {
if (hasNext) { if (hasNext) {
SBlock* pBlock = getCurrentBlock(pBlockIter); SDataBlk* pBlock = getCurrentBlock(pBlockIter);
int32_t numOfRows = pBlock->nRow; int32_t numOfRows = pBlock->nRow;
pTableBlockInfo->totalRows += numOfRows; pTableBlockInfo->totalRows += numOfRows;

136
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -433,11 +433,11 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
goto _err; goto _err;
} }
// last // sst
for (int32_t iLast = 0; iLast < pSet->nLastF; iLast++) { for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aLastF[iLast], fname); tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[iSst], fname);
pReader->aLastFD[iLast] = taosOpenFile(fname, TD_FILE_READ); pReader->aLastFD[iSst] = taosOpenFile(fname, TD_FILE_READ);
if (pReader->aLastFD[iLast] == NULL) { if (pReader->aLastFD[iSst] == NULL) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
@ -474,9 +474,9 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) {
goto _err; goto _err;
} }
// last // sst
for (int32_t iLast = 0; iLast < (*ppReader)->pSet->nLastF; iLast++) { for (int32_t iSst = 0; iSst < (*ppReader)->pSet->nSstF; iSst++) {
if (taosCloseFile(&(*ppReader)->aLastFD[iLast]) < 0) { if (taosCloseFile(&(*ppReader)->aLastFD[iSst]) < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
@ -559,14 +559,14 @@ _err:
return code; return code;
} }
int32_t tsdbReadBlockL(SDataFReader *pReader, int32_t iLast, SArray *aBlockL) { int32_t tsdbReadSstBlk(SDataFReader *pReader, int32_t iSst, SArray *aSstBlk) {
int32_t code = 0; int32_t code = 0;
int64_t offset = pReader->pSet->aLastF[iLast]->offset; int64_t offset = pReader->pSet->aSstF[iSst]->offset;
int64_t size = pReader->pSet->aLastF[iLast]->size - offset; int64_t size = pReader->pSet->aSstF[iSst]->size - offset;
int64_t n; int64_t n;
uint32_t delimiter; uint32_t delimiter;
taosArrayClear(aBlockL); taosArrayClear(aSstBlk);
if (size == 0) { if (size == 0) {
goto _exit; goto _exit;
} }
@ -576,13 +576,13 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, int32_t iLast, SArray *aBlockL) {
if (code) goto _err; if (code) goto _err;
// seek // seek
if (taosLSeekFile(pReader->aLastFD[iLast], offset, SEEK_SET) < 0) { if (taosLSeekFile(pReader->aLastFD[iSst], offset, SEEK_SET) < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
// read // read
n = taosReadFile(pReader->aLastFD[iLast], pReader->aBuf[0], size); n = taosReadFile(pReader->aLastFD[iSst], pReader->aBuf[0], size);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
@ -603,10 +603,10 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, int32_t iLast, SArray *aBlockL) {
ASSERT(delimiter == TSDB_FILE_DLMT); ASSERT(delimiter == TSDB_FILE_DLMT);
while (n < size - sizeof(TSCKSUM)) { while (n < size - sizeof(TSCKSUM)) {
SBlockL blockl; SSstBlk blockl;
n += tGetBlockL(pReader->aBuf[0] + n, &blockl); n += tGetSstBlk(pReader->aBuf[0] + n, &blockl);
if (taosArrayPush(aBlockL, &blockl) == NULL) { if (taosArrayPush(aSstBlk, &blockl) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
@ -677,9 +677,9 @@ _err:
return code; return code;
} }
int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg) { int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pDataBlk, SArray *aColumnDataAgg) {
int32_t code = 0; int32_t code = 0;
SSmaInfo *pSmaInfo = &pBlock->smaInfo; SSmaInfo *pSmaInfo = &pDataBlk->smaInfo;
ASSERT(pSmaInfo->size > 0); ASSERT(pSmaInfo->size > 0);
@ -843,13 +843,13 @@ _err:
return code; return code;
} }
int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData) { int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData) {
int32_t code = 0; int32_t code = 0;
code = tsdbReadBlockDataImpl(pReader, &pBlock->aSubBlock[0], 0, pBlockData); code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[0], 0, pBlockData);
if (code) goto _err; if (code) goto _err;
if (pBlock->nSubBlock > 1) { if (pDataBlk->nSubBlock > 1) {
SBlockData bData1; SBlockData bData1;
SBlockData bData2; SBlockData bData2;
@ -863,8 +863,8 @@ int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBl
tBlockDataInitEx(&bData1, pBlockData); tBlockDataInitEx(&bData1, pBlockData);
tBlockDataInitEx(&bData2, pBlockData); tBlockDataInitEx(&bData2, pBlockData);
for (int32_t iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { for (int32_t iSubBlock = 1; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) {
code = tsdbReadBlockDataImpl(pReader, &pBlock->aSubBlock[iSubBlock], 0, &bData1); code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[iSubBlock], 0, &bData1);
if (code) { if (code) {
tBlockDataDestroy(&bData1, 1); tBlockDataDestroy(&bData1, 1);
tBlockDataDestroy(&bData2, 1); tBlockDataDestroy(&bData2, 1);
@ -897,10 +897,10 @@ _err:
return code; return code;
} }
int32_t tsdbReadLastBlock(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, SBlockData *pBlockData) { int32_t tsdbReadSstBlock(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData) {
int32_t code = 0; int32_t code = 0;
code = tsdbReadBlockDataImpl(pReader, &pBlockL->bInfo, 1, pBlockData); code = tsdbReadBlockDataImpl(pReader, &pSstBlk->bInfo, 1, pBlockData);
if (code) goto _err; if (code) goto _err;
return code; return code;
@ -910,15 +910,15 @@ _err:
return code; return code;
} }
int32_t tsdbReadLastBlockEx(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, SBlockData *pBlockData) { int32_t tsdbReadSstBlockEx(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData) {
int32_t code = 0; int32_t code = 0;
// read // read
code = tsdbReadAndCheck(pReader->aLastFD[iLast], pBlockL->bInfo.offset, &pReader->aBuf[0], pBlockL->bInfo.szBlock, 0); code = tsdbReadAndCheck(pReader->aLastFD[iSst], pSstBlk->bInfo.offset, &pReader->aBuf[0], pSstBlk->bInfo.szBlock, 0);
if (code) goto _exit; if (code) goto _exit;
// decmpr // decmpr
code = tDecmprBlockData(pReader->aBuf[0], pBlockL->bInfo.szBlock, pBlockData, &pReader->aBuf[1]); code = tDecmprBlockData(pReader->aBuf[0], pSstBlk->bInfo.szBlock, pBlockData, &pReader->aBuf[1]);
if (code) goto _exit; if (code) goto _exit;
_exit: _exit:
@ -947,14 +947,14 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS
.pHeadF = &pWriter->fHead, .pHeadF = &pWriter->fHead,
.pDataF = &pWriter->fData, .pDataF = &pWriter->fData,
.pSmaF = &pWriter->fSma, .pSmaF = &pWriter->fSma,
.nLastF = pSet->nLastF // .nSstF = pSet->nSstF //
}; };
pWriter->fHead = *pSet->pHeadF; pWriter->fHead = *pSet->pHeadF;
pWriter->fData = *pSet->pDataF; pWriter->fData = *pSet->pDataF;
pWriter->fSma = *pSet->pSmaF; pWriter->fSma = *pSet->pSmaF;
for (int8_t iLast = 0; iLast < pSet->nLastF; iLast++) { for (int8_t iSst = 0; iSst < pSet->nSstF; iSst++) {
pWriter->wSet.aLastF[iLast] = &pWriter->fLast[iLast]; pWriter->wSet.aSstF[iSst] = &pWriter->fSst[iSst];
pWriter->fLast[iLast] = *pSet->aLastF[iLast]; pWriter->fSst[iSst] = *pSet->aSstF[iSst];
} }
// head // head
@ -1036,10 +1036,10 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS
ASSERT(n == pWriter->fSma.size); ASSERT(n == pWriter->fSma.size);
} }
// last // sst
ASSERT(pWriter->fLast[pSet->nLastF - 1].size == 0); ASSERT(pWriter->fSst[pSet->nSstF - 1].size == 0);
flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
tsdbLastFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fLast[pSet->nLastF - 1], fname); tsdbSstFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fSst[pSet->nSstF - 1], fname);
pWriter->pLastFD = taosOpenFile(fname, flag); pWriter->pLastFD = taosOpenFile(fname, flag);
if (pWriter->pLastFD == NULL) { if (pWriter->pLastFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
@ -1050,7 +1050,7 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
pWriter->fLast[pWriter->wSet.nLastF - 1].size += TSDB_FHDR_SIZE; pWriter->fSst[pWriter->wSet.nSstF - 1].size += TSDB_FHDR_SIZE;
*ppWriter = pWriter; *ppWriter = pWriter;
return code; return code;
@ -1179,9 +1179,9 @@ int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter) {
goto _err; goto _err;
} }
// last ============== // sst ==============
memset(hdr, 0, TSDB_FHDR_SIZE); memset(hdr, 0, TSDB_FHDR_SIZE);
tPutLastFile(hdr, &pWriter->fLast[pWriter->wSet.nLastF - 1]); tPutSstFile(hdr, &pWriter->fSst[pWriter->wSet.nSstF - 1]);
taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE);
n = taosLSeekFile(pWriter->pLastFD, 0, SEEK_SET); n = taosLSeekFile(pWriter->pLastFD, 0, SEEK_SET);
@ -1301,22 +1301,22 @@ _err:
return code; return code;
} }
int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) { int32_t tsdbWriteSstBlk(SDataFWriter *pWriter, SArray *aSstBlk) {
int32_t code = 0; int32_t code = 0;
SLastFile *pLastFile = &pWriter->fLast[pWriter->wSet.nLastF - 1]; SSstFile *pSstFile = &pWriter->fSst[pWriter->wSet.nSstF - 1];
int64_t size; int64_t size;
int64_t n; int64_t n;
// check // check
if (taosArrayGetSize(aBlockL) == 0) { if (taosArrayGetSize(aSstBlk) == 0) {
pLastFile->offset = pLastFile->size; pSstFile->offset = pSstFile->size;
goto _exit; goto _exit;
} }
// size // size
size = sizeof(uint32_t); // TSDB_FILE_DLMT size = sizeof(uint32_t); // TSDB_FILE_DLMT
for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aBlockL); iBlockL++) { for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSstBlk); iBlockL++) {
size += tPutBlockL(NULL, taosArrayGet(aBlockL, iBlockL)); size += tPutSstBlk(NULL, taosArrayGet(aSstBlk, iBlockL));
} }
size += sizeof(TSCKSUM); size += sizeof(TSCKSUM);
@ -1327,8 +1327,8 @@ int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) {
// encode // encode
n = 0; n = 0;
n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT);
for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aBlockL); iBlockL++) { for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSstBlk); iBlockL++) {
n += tPutBlockL(pWriter->aBuf[0] + n, taosArrayGet(aBlockL, iBlockL)); n += tPutSstBlk(pWriter->aBuf[0] + n, taosArrayGet(aSstBlk, iBlockL));
} }
taosCalcChecksumAppend(0, pWriter->aBuf[0], size); taosCalcChecksumAppend(0, pWriter->aBuf[0], size);
@ -1342,12 +1342,12 @@ int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) {
} }
// update // update
pLastFile->offset = pLastFile->size; pSstFile->offset = pSstFile->size;
pLastFile->size += size; pSstFile->size += size;
_exit: _exit:
tsdbTrace("vgId:%d tsdb write blockl, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), tsdbTrace("vgId:%d tsdb write blockl, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode),
pLastFile->offset, size); pSstFile->offset, size);
return code; return code;
_err: _err:
@ -1355,28 +1355,28 @@ _err:
return code; return code;
} }
static void tsdbUpdateBlockInfo(SBlockData *pBlockData, SBlock *pBlock) { static void tsdbUpdateBlockInfo(SBlockData *pBlockData, SDataBlk *pDataBlk) {
for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) {
TSDBKEY key = {.ts = pBlockData->aTSKEY[iRow], .version = pBlockData->aVersion[iRow]}; TSDBKEY key = {.ts = pBlockData->aTSKEY[iRow], .version = pBlockData->aVersion[iRow]};
if (iRow == 0) { if (iRow == 0) {
if (tsdbKeyCmprFn(&pBlock->minKey, &key) > 0) { if (tsdbKeyCmprFn(&pDataBlk->minKey, &key) > 0) {
pBlock->minKey = key; pDataBlk->minKey = key;
} }
} else { } else {
if (pBlockData->aTSKEY[iRow] == pBlockData->aTSKEY[iRow - 1]) { if (pBlockData->aTSKEY[iRow] == pBlockData->aTSKEY[iRow - 1]) {
pBlock->hasDup = 1; pDataBlk->hasDup = 1;
} }
} }
if (iRow == pBlockData->nRow - 1 && tsdbKeyCmprFn(&pBlock->maxKey, &key) < 0) { if (iRow == pBlockData->nRow - 1 && tsdbKeyCmprFn(&pDataBlk->maxKey, &key) < 0) {
pBlock->maxKey = key; pDataBlk->maxKey = key;
} }
pBlock->minVer = TMIN(pBlock->minVer, key.version); pDataBlk->minVer = TMIN(pDataBlk->minVer, key.version);
pBlock->maxVer = TMAX(pBlock->maxVer, key.version); pDataBlk->maxVer = TMAX(pDataBlk->maxVer, key.version);
} }
pBlock->nRow += pBlockData->nRow; pDataBlk->nRow += pBlockData->nRow;
} }
static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, SSmaInfo *pSmaInfo) { static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, SSmaInfo *pSmaInfo) {
@ -1431,7 +1431,7 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock
ASSERT(pBlockData->nRow > 0); ASSERT(pBlockData->nRow > 0);
pBlkInfo->offset = toLast ? pWriter->fLast[pWriter->wSet.nLastF - 1].size : pWriter->fData.size; pBlkInfo->offset = toLast ? pWriter->fSst[pWriter->wSet.nSstF - 1].size : pWriter->fData.size;
pBlkInfo->szBlock = 0; pBlkInfo->szBlock = 0;
pBlkInfo->szKey = 0; pBlkInfo->szKey = 0;
@ -1475,7 +1475,7 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock
// update info // update info
if (toLast) { if (toLast) {
pWriter->fLast[pWriter->wSet.nLastF - 1].size += pBlkInfo->szBlock; pWriter->fSst[pWriter->wSet.nSstF - 1].size += pBlkInfo->szBlock;
} else { } else {
pWriter->fData.size += pBlkInfo->szBlock; pWriter->fData.size += pBlkInfo->szBlock;
} }
@ -1554,9 +1554,9 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
taosCloseFile(&pOutFD); taosCloseFile(&pOutFD);
taosCloseFile(&PInFD); taosCloseFile(&PInFD);
// last // sst
tsdbLastFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aLastF[0], fNameFrom); tsdbSstFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSstF[0], fNameFrom);
tsdbLastFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aLastF[0], fNameTo); tsdbSstFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSstF[0], fNameTo);
pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
if (pOutFD == NULL) { if (pOutFD == NULL) {
@ -1570,7 +1570,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
goto _err; goto _err;
} }
n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->aLastF[0]->size); n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->aSstF[0]->size);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;

2
source/dnode/vnode/src/tsdb/tsdbRetention.c
View File

@ -60,7 +60,7 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) {
if (expLevel < 0) { if (expLevel < 0) {
taosMemoryFree(pSet->pHeadF); taosMemoryFree(pSet->pHeadF);
taosMemoryFree(pSet->pDataF); taosMemoryFree(pSet->pDataF);
taosMemoryFree(pSet->aLastF[0]); taosMemoryFree(pSet->aSstF[0]);
taosMemoryFree(pSet->pSmaF); taosMemoryFree(pSet->pSmaF);
taosArrayRemove(fs.aDFileSet, iSet); taosArrayRemove(fs.aDFileSet, iSet);
iSet--; iSet--;

116
source/dnode/vnode/src/tsdb/tsdbSnapshot.c
View File

@ -27,13 +27,13 @@ struct STsdbSnapReader {
int32_t fid; int32_t fid;
SDataFReader* pDataFReader; SDataFReader* pDataFReader;
SArray* aBlockIdx; // SArray<SBlockIdx> SArray* aBlockIdx; // SArray<SBlockIdx>
SArray* aBlockL; // SArray<SBlockL> SArray* aSstBlk; // SArray<SSstBlk>
SBlockIdx* pBlockIdx; SBlockIdx* pBlockIdx;
SBlockL* pBlockL; SSstBlk* pSstBlk;
int32_t iBlockIdx; int32_t iBlockIdx;
int32_t iBlockL; int32_t iBlockL;
SMapData mBlock; // SMapData<SBlock> SMapData mBlock; // SMapData<SDataBlk>
int32_t iBlock; int32_t iBlock;
SBlockData oBlockData; SBlockData oBlockData;
SBlockData nBlockData; SBlockData nBlockData;
@ -64,7 +64,7 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx); code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx);
if (code) goto _err; if (code) goto _err;
code = tsdbReadBlockL(pReader->pDataFReader, 0, pReader->aBlockL); code = tsdbReadSstBlk(pReader->pDataFReader, 0, pReader->aSstBlk);
if (code) goto _err; if (code) goto _err;
// init // init
@ -82,13 +82,13 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
pReader->iBlockL = 0; pReader->iBlockL = 0;
while (true) { while (true) {
if (pReader->iBlockL >= taosArrayGetSize(pReader->aBlockL)) { if (pReader->iBlockL >= taosArrayGetSize(pReader->aSstBlk)) {
pReader->pBlockL = NULL; pReader->pSstBlk = NULL;
break; break;
} }
pReader->pBlockL = (SBlockL*)taosArrayGet(pReader->aBlockL, pReader->iBlockL); pReader->pSstBlk = (SSstBlk*)taosArrayGet(pReader->aSstBlk, pReader->iBlockL);
if (pReader->pBlockL->minVer <= pReader->ever && pReader->pBlockL->maxVer >= pReader->sver) { if (pReader->pSstBlk->minVer <= pReader->ever && pReader->pSstBlk->maxVer >= pReader->sver) {
// TODO // TODO
break; break;
} }
@ -101,8 +101,8 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
} }
while (true) { while (true) {
if (pReader->pBlockIdx && pReader->pBlockL) { if (pReader->pBlockIdx && pReader->pSstBlk) {
TABLEID id = {.suid = pReader->pBlockL->suid, .uid = pReader->pBlockL->minUid}; TABLEID id = {.suid = pReader->pSstBlk->suid, .uid = pReader->pSstBlk->minUid};
ASSERT(0); ASSERT(0);
@ -115,8 +115,8 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
// } // }
} else if (pReader->pBlockIdx) { } else if (pReader->pBlockIdx) {
while (pReader->iBlock < pReader->mBlock.nItem) { while (pReader->iBlock < pReader->mBlock.nItem) {
SBlock block; SDataBlk block;
tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, &block, tGetBlock); tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, &block, tGetDataBlk);
if (block.minVer <= pReader->ever && block.maxVer >= pReader->sver) { if (block.minVer <= pReader->ever && block.maxVer >= pReader->sver) {
// load data (todo) // load data (todo)
@ -142,18 +142,18 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
} }
if (*ppData) goto _exit; if (*ppData) goto _exit;
} else if (pReader->pBlockL) { } else if (pReader->pSstBlk) {
while (pReader->pBlockL) { while (pReader->pSstBlk) {
if (pReader->pBlockL->minVer <= pReader->ever && pReader->pBlockL->maxVer >= pReader->sver) { if (pReader->pSstBlk->minVer <= pReader->ever && pReader->pSstBlk->maxVer >= pReader->sver) {
// load data (todo) // load data (todo)
} }
// next // next
pReader->iBlockL++; pReader->iBlockL++;
if (pReader->iBlockL < taosArrayGetSize(pReader->aBlockL)) { if (pReader->iBlockL < taosArrayGetSize(pReader->aSstBlk)) {
pReader->pBlockL = (SBlockL*)taosArrayGetSize(pReader->aBlockL); pReader->pSstBlk = (SSstBlk*)taosArrayGetSize(pReader->aSstBlk);
} else { } else {
pReader->pBlockL = NULL; pReader->pSstBlk = NULL;
} }
if (*ppData) goto _exit; if (*ppData) goto _exit;
@ -298,8 +298,8 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
pReader->aBlockL = taosArrayInit(0, sizeof(SBlockL)); pReader->aSstBlk = taosArrayInit(0, sizeof(SSstBlk));
if (pReader->aBlockL == NULL) { if (pReader->aSstBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
@ -338,7 +338,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) {
if (pReader->pDataFReader) { if (pReader->pDataFReader) {
tsdbDataFReaderClose(&pReader->pDataFReader); tsdbDataFReaderClose(&pReader->pDataFReader);
} }
taosArrayDestroy(pReader->aBlockL); taosArrayDestroy(pReader->aSstBlk);
taosArrayDestroy(pReader->aBlockIdx); taosArrayDestroy(pReader->aBlockIdx);
tMapDataClear(&pReader->mBlock); tMapDataClear(&pReader->mBlock);
tBlockDataDestroy(&pReader->oBlockData, 1); tBlockDataDestroy(&pReader->oBlockData, 1);
@ -426,24 +426,24 @@ struct STsdbSnapWriter {
SArray* aBlockIdx; // SArray<SBlockIdx> SArray* aBlockIdx; // SArray<SBlockIdx>
int32_t iBlockIdx; int32_t iBlockIdx;
SBlockIdx* pBlockIdx; SBlockIdx* pBlockIdx;
SMapData mBlock; // SMapData<SBlock> SMapData mBlock; // SMapData<SDataBlk>
int32_t iBlock; int32_t iBlock;
SBlockData* pBlockData; SBlockData* pBlockData;
int32_t iRow; int32_t iRow;
SBlockData bDataR; SBlockData bDataR;
SArray* aBlockL; // SArray<SBlockL> SArray* aSstBlk; // SArray<SSstBlk>
int32_t iBlockL; int32_t iBlockL;
SBlockData lDataR; SBlockData lDataR;
SDataFWriter* pDataFWriter; SDataFWriter* pDataFWriter;
SBlockIdx* pBlockIdxW; // NULL when no committing table SBlockIdx* pBlockIdxW; // NULL when no committing table
SBlock blockW; SDataBlk blockW;
SBlockData bDataW; SBlockData bDataW;
SBlockIdx blockIdxW; SBlockIdx blockIdxW;
SMapData mBlockW; // SMapData<SBlock> SMapData mBlockW; // SMapData<SDataBlk>
SArray* aBlockIdxW; // SArray<SBlockIdx> SArray* aBlockIdxW; // SArray<SBlockIdx>
SArray* aBlockLW; // SArray<SBlockL> SArray* aBlockLW; // SArray<SSstBlk>
// for del file // for del file
SDelFReader* pDelFReader; SDelFReader* pDelFReader;
@ -475,10 +475,10 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) {
// &pWriter->blockW, pWriter->cmprAlg); // &pWriter->blockW, pWriter->cmprAlg);
if (code) goto _err; if (code) goto _err;
code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk);
if (code) goto _err; if (code) goto _err;
tBlockReset(&pWriter->blockW); tDataBlkReset(&pWriter->blockW);
tBlockDataClear(&pWriter->bDataW); tBlockDataClear(&pWriter->bDataW);
} }
@ -499,15 +499,15 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) {
// &pWriter->blockW, pWriter->cmprAlg); // &pWriter->blockW, pWriter->cmprAlg);
// if (code) goto _err; // if (code) goto _err;
code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk);
if (code) goto _err; if (code) goto _err;
} }
while (true) { while (true) {
if (pWriter->iBlock >= pWriter->mBlock.nItem) break; if (pWriter->iBlock >= pWriter->mBlock.nItem) break;
SBlock block; SDataBlk block;
tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock); tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetDataBlk);
// if (block.last) { // if (block.last) {
// code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); // code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, &pWriter->bDataR, NULL, NULL);
@ -520,13 +520,13 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) {
// if (code) goto _err; // if (code) goto _err;
// } // }
code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutDataBlk);
if (code) goto _err; if (code) goto _err;
pWriter->iBlock++; pWriter->iBlock++;
} }
// SBlock // SDataBlk
// code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, pWriter->pBlockIdxW); // code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, pWriter->pBlockIdxW);
// if (code) goto _err; // if (code) goto _err;
@ -553,10 +553,10 @@ static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* p
if (code) goto _err; if (code) goto _err;
// SBlockData // SBlockData
SBlock block; SDataBlk block;
tMapDataReset(&pWriter->mBlockW); tMapDataReset(&pWriter->mBlockW);
for (int32_t iBlock = 0; iBlock < pWriter->mBlock.nItem; iBlock++) { for (int32_t iBlock = 0; iBlock < pWriter->mBlock.nItem; iBlock++) {
tMapDataGetItemByIdx(&pWriter->mBlock, iBlock, &block, tGetBlock); tMapDataGetItemByIdx(&pWriter->mBlock, iBlock, &block, tGetDataBlk);
// if (block.last) { // if (block.last) {
// code = tsdbReadBlockData(pWriter->pDataFReader, pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); // code = tsdbReadBlockData(pWriter->pDataFReader, pBlockIdx, &block, &pWriter->bDataR, NULL, NULL);
@ -570,11 +570,11 @@ static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* p
// if (code) goto _err; // if (code) goto _err;
// } // }
code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutDataBlk);
if (code) goto _err; if (code) goto _err;
} }
// SBlock // SDataBlk
SBlockIdx blockIdx = {.suid = pBlockIdx->suid, .uid = pBlockIdx->uid}; SBlockIdx blockIdx = {.suid = pBlockIdx->suid, .uid = pBlockIdx->uid};
code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, &blockIdx); code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, &blockIdx);
if (code) goto _err; if (code) goto _err;
@ -642,10 +642,10 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) {
while (true) { while (true) {
if (pWriter->iBlock >= pWriter->mBlock.nItem) break; if (pWriter->iBlock >= pWriter->mBlock.nItem) break;
SBlock block; SDataBlk block;
int32_t c; int32_t c;
tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock); tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetDataBlk);
// if (block.last) { // if (block.last) {
// pWriter->pBlockData = &pWriter->bDataR; // pWriter->pBlockData = &pWriter->bDataR;
@ -668,14 +668,14 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) {
// &pWriter->blockW, pWriter->cmprAlg); // &pWriter->blockW, pWriter->cmprAlg);
// if (code) goto _err; // if (code) goto _err;
code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk);
if (code) goto _err; if (code) goto _err;
tBlockReset(&pWriter->blockW); tDataBlkReset(&pWriter->blockW);
tBlockDataClear(&pWriter->bDataW); tBlockDataClear(&pWriter->bDataW);
} }
code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutDataBlk);
if (code) goto _err; if (code) goto _err;
pWriter->iBlock++; pWriter->iBlock++;
@ -719,10 +719,10 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) {
// &pWriter->blockW, pWriter->cmprAlg); // &pWriter->blockW, pWriter->cmprAlg);
// if (code) goto _err; // if (code) goto _err;
code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk);
if (code) goto _err; if (code) goto _err;
tBlockReset(&pWriter->blockW); tDataBlkReset(&pWriter->blockW);
tBlockDataClear(&pWriter->bDataW); tBlockDataClear(&pWriter->bDataW);
} }
@ -803,7 +803,7 @@ static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, TABLEID id) {
pWriter->pBlockIdxW->suid = id.suid; pWriter->pBlockIdxW->suid = id.suid;
pWriter->pBlockIdxW->uid = id.uid; pWriter->pBlockIdxW->uid = id.uid;
tBlockReset(&pWriter->blockW); tDataBlkReset(&pWriter->blockW);
tBlockDataReset(&pWriter->bDataW); tBlockDataReset(&pWriter->bDataW);
tMapDataReset(&pWriter->mBlockW); tMapDataReset(&pWriter->mBlockW);
} }
@ -845,7 +845,7 @@ static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) {
// write remain stuff // write remain stuff
if (taosArrayGetSize(pWriter->aBlockLW) > 0) { if (taosArrayGetSize(pWriter->aBlockLW) > 0) {
code = tsdbWriteBlockL(pWriter->pDataFWriter, pWriter->aBlockIdxW); code = tsdbWriteSstBlk(pWriter->pDataFWriter, pWriter->aBlockIdxW);
if (code) goto _err; if (code) goto _err;
} }
@ -911,12 +911,12 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx); code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx);
if (code) goto _err; if (code) goto _err;
code = tsdbReadBlockL(pWriter->pDataFReader, 0, pWriter->aBlockL); code = tsdbReadSstBlk(pWriter->pDataFReader, 0, pWriter->aSstBlk);
if (code) goto _err; if (code) goto _err;
} else { } else {
ASSERT(pWriter->pDataFReader == NULL); ASSERT(pWriter->pDataFReader == NULL);
taosArrayClear(pWriter->aBlockIdx); taosArrayClear(pWriter->aBlockIdx);
taosArrayClear(pWriter->aBlockL); taosArrayClear(pWriter->aSstBlk);
} }
pWriter->iBlockIdx = 0; pWriter->iBlockIdx = 0;
pWriter->pBlockIdx = NULL; pWriter->pBlockIdx = NULL;
@ -931,25 +931,25 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
// write // write
SHeadFile fHead; SHeadFile fHead;
SDataFile fData; SDataFile fData;
SLastFile fLast; SSstFile fLast;
SSmaFile fSma; SSmaFile fSma;
SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .aLastF[0] = &fLast, .pSmaF = &fSma}; SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .aSstF[0] = &fLast, .pSmaF = &fSma};
if (pSet) { if (pSet) {
wSet.diskId = pSet->diskId; wSet.diskId = pSet->diskId;
wSet.fid = fid; wSet.fid = fid;
wSet.nLastF = 1; wSet.nSstF = 1;
fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0}; fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0};
fData = *pSet->pDataF; fData = *pSet->pDataF;
fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0}; fLast = (SSstFile){.commitID = pWriter->commitID, .size = 0};
fSma = *pSet->pSmaF; fSma = *pSet->pSmaF;
} else { } else {
wSet.diskId = (SDiskID){.level = 0, .id = 0}; wSet.diskId = (SDiskID){.level = 0, .id = 0};
wSet.fid = fid; wSet.fid = fid;
wSet.nLastF = 1; wSet.nSstF = 1;
fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0}; fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0};
fData = (SDataFile){.commitID = pWriter->commitID, .size = 0}; fData = (SDataFile){.commitID = pWriter->commitID, .size = 0};
fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0, .offset = 0}; fLast = (SSstFile){.commitID = pWriter->commitID, .size = 0, .offset = 0};
fSma = (SSmaFile){.commitID = pWriter->commitID, .size = 0}; fSma = (SSmaFile){.commitID = pWriter->commitID, .size = 0};
} }
@ -1147,8 +1147,8 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr
code = tBlockDataCreate(&pWriter->bDataR); code = tBlockDataCreate(&pWriter->bDataR);
if (code) goto _err; if (code) goto _err;
pWriter->aBlockL = taosArrayInit(0, sizeof(SBlockL)); pWriter->aSstBlk = taosArrayInit(0, sizeof(SSstBlk));
if (pWriter->aBlockL == NULL) { if (pWriter->aSstBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
@ -1161,7 +1161,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr
code = tBlockDataCreate(&pWriter->bDataW); code = tBlockDataCreate(&pWriter->bDataW);
if (code) goto _err; if (code) goto _err;
pWriter->aBlockLW = taosArrayInit(0, sizeof(SBlockL)); pWriter->aBlockLW = taosArrayInit(0, sizeof(SSstBlk));
if (pWriter->aBlockLW == NULL) { if (pWriter->aBlockLW == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;

156
source/dnode/vnode/src/tsdb/tsdbUtil.c
View File

@ -214,7 +214,7 @@ int32_t tCmprBlockIdx(void const *lhs, void const *rhs) {
int32_t tCmprBlockL(void const *lhs, void const *rhs) { int32_t tCmprBlockL(void const *lhs, void const *rhs) {
SBlockIdx *lBlockIdx = (SBlockIdx *)lhs; SBlockIdx *lBlockIdx = (SBlockIdx *)lhs;
SBlockL *rBlockL = (SBlockL *)rhs; SSstBlk *rBlockL = (SSstBlk *)rhs;
if (lBlockIdx->suid < rBlockL->suid) { if (lBlockIdx->suid < rBlockL->suid) {
return -1; return -1;
@ -231,69 +231,69 @@ int32_t tCmprBlockL(void const *lhs, void const *rhs) {
return 0; return 0;
} }
// SBlock ====================================================== // SDataBlk ======================================================
void tBlockReset(SBlock *pBlock) { void tDataBlkReset(SDataBlk *pDataBlk) {
*pBlock = (SBlock){.minKey = TSDBKEY_MAX, .maxKey = TSDBKEY_MIN, .minVer = VERSION_MAX, .maxVer = VERSION_MIN}; *pDataBlk = (SDataBlk){.minKey = TSDBKEY_MAX, .maxKey = TSDBKEY_MIN, .minVer = VERSION_MAX, .maxVer = VERSION_MIN};
} }
int32_t tPutBlock(uint8_t *p, void *ph) { int32_t tPutDataBlk(uint8_t *p, void *ph) {
int32_t n = 0; int32_t n = 0;
SBlock *pBlock = (SBlock *)ph; SDataBlk *pDataBlk = (SDataBlk *)ph;
n += tPutI64v(p ? p + n : p, pBlock->minKey.version); n += tPutI64v(p ? p + n : p, pDataBlk->minKey.version);
n += tPutI64v(p ? p + n : p, pBlock->minKey.ts); n += tPutI64v(p ? p + n : p, pDataBlk->minKey.ts);
n += tPutI64v(p ? p + n : p, pBlock->maxKey.version); n += tPutI64v(p ? p + n : p, pDataBlk->maxKey.version);
n += tPutI64v(p ? p + n : p, pBlock->maxKey.ts); n += tPutI64v(p ? p + n : p, pDataBlk->maxKey.ts);
n += tPutI64v(p ? p + n : p, pBlock->minVer); n += tPutI64v(p ? p + n : p, pDataBlk->minVer);
n += tPutI64v(p ? p + n : p, pBlock->maxVer); n += tPutI64v(p ? p + n : p, pDataBlk->maxVer);
n += tPutI32v(p ? p + n : p, pBlock->nRow); n += tPutI32v(p ? p + n : p, pDataBlk->nRow);
n += tPutI8(p ? p + n : p, pBlock->hasDup); n += tPutI8(p ? p + n : p, pDataBlk->hasDup);
n += tPutI8(p ? p + n : p, pBlock->nSubBlock); n += tPutI8(p ? p + n : p, pDataBlk->nSubBlock);
for (int8_t iSubBlock = 0; iSubBlock < pBlock->nSubBlock; iSubBlock++) { for (int8_t iSubBlock = 0; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) {
n += tPutI64v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].offset); n += tPutI64v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].offset);
n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].szBlock); n += tPutI32v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].szBlock);
n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].szKey); n += tPutI32v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].szKey);
} }
if (pBlock->nSubBlock == 1 && !pBlock->hasDup) { if (pDataBlk->nSubBlock == 1 && !pDataBlk->hasDup) {
n += tPutI64v(p ? p + n : p, pBlock->smaInfo.offset); n += tPutI64v(p ? p + n : p, pDataBlk->smaInfo.offset);
n += tPutI32v(p ? p + n : p, pBlock->smaInfo.size); n += tPutI32v(p ? p + n : p, pDataBlk->smaInfo.size);
} }
return n; return n;
} }
int32_t tGetBlock(uint8_t *p, void *ph) { int32_t tGetDataBlk(uint8_t *p, void *ph) {
int32_t n = 0; int32_t n = 0;
SBlock *pBlock = (SBlock *)ph; SDataBlk *pDataBlk = (SDataBlk *)ph;
n += tGetI64v(p + n, &pBlock->minKey.version); n += tGetI64v(p + n, &pDataBlk->minKey.version);
n += tGetI64v(p + n, &pBlock->minKey.ts); n += tGetI64v(p + n, &pDataBlk->minKey.ts);
n += tGetI64v(p + n, &pBlock->maxKey.version); n += tGetI64v(p + n, &pDataBlk->maxKey.version);
n += tGetI64v(p + n, &pBlock->maxKey.ts); n += tGetI64v(p + n, &pDataBlk->maxKey.ts);
n += tGetI64v(p + n, &pBlock->minVer); n += tGetI64v(p + n, &pDataBlk->minVer);
n += tGetI64v(p + n, &pBlock->maxVer); n += tGetI64v(p + n, &pDataBlk->maxVer);
n += tGetI32v(p + n, &pBlock->nRow); n += tGetI32v(p + n, &pDataBlk->nRow);
n += tGetI8(p + n, &pBlock->hasDup); n += tGetI8(p + n, &pDataBlk->hasDup);
n += tGetI8(p + n, &pBlock->nSubBlock); n += tGetI8(p + n, &pDataBlk->nSubBlock);
for (int8_t iSubBlock = 0; iSubBlock < pBlock->nSubBlock; iSubBlock++) { for (int8_t iSubBlock = 0; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) {
n += tGetI64v(p + n, &pBlock->aSubBlock[iSubBlock].offset); n += tGetI64v(p + n, &pDataBlk->aSubBlock[iSubBlock].offset);
n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].szBlock); n += tGetI32v(p + n, &pDataBlk->aSubBlock[iSubBlock].szBlock);
n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].szKey); n += tGetI32v(p + n, &pDataBlk->aSubBlock[iSubBlock].szKey);
} }
if (pBlock->nSubBlock == 1 && !pBlock->hasDup) { if (pDataBlk->nSubBlock == 1 && !pDataBlk->hasDup) {
n += tGetI64v(p + n, &pBlock->smaInfo.offset); n += tGetI64v(p + n, &pDataBlk->smaInfo.offset);
n += tGetI32v(p + n, &pBlock->smaInfo.size); n += tGetI32v(p + n, &pDataBlk->smaInfo.size);
} else { } else {
pBlock->smaInfo.offset = 0; pDataBlk->smaInfo.offset = 0;
pBlock->smaInfo.size = 0; pDataBlk->smaInfo.size = 0;
} }
return n; return n;
} }
int32_t tBlockCmprFn(const void *p1, const void *p2) { int32_t tDataBlkCmprFn(const void *p1, const void *p2) {
SBlock *pBlock1 = (SBlock *)p1; SDataBlk *pBlock1 = (SDataBlk *)p1;
SBlock *pBlock2 = (SBlock *)p2; SDataBlk *pBlock2 = (SDataBlk *)p2;
if (tsdbKeyCmprFn(&pBlock1->maxKey, &pBlock2->minKey) < 0) { if (tsdbKeyCmprFn(&pBlock1->maxKey, &pBlock2->minKey) < 0) {
return -1; return -1;
@ -304,48 +304,48 @@ int32_t tBlockCmprFn(const void *p1, const void *p2) {
return 0; return 0;
} }
bool tBlockHasSma(SBlock *pBlock) { bool tDataBlkHasSma(SDataBlk *pDataBlk) {
if (pBlock->nSubBlock > 1) return false; if (pDataBlk->nSubBlock > 1) return false;
if (pBlock->hasDup) return false; if (pDataBlk->hasDup) return false;
return pBlock->smaInfo.size > 0; return pDataBlk->smaInfo.size > 0;
} }
// SBlockL ====================================================== // SSstBlk ======================================================
int32_t tPutBlockL(uint8_t *p, void *ph) { int32_t tPutSstBlk(uint8_t *p, void *ph) {
int32_t n = 0; int32_t n = 0;
SBlockL *pBlockL = (SBlockL *)ph; SSstBlk *pSstBlk = (SSstBlk *)ph;
n += tPutI64(p ? p + n : p, pBlockL->suid); n += tPutI64(p ? p + n : p, pSstBlk->suid);
n += tPutI64(p ? p + n : p, pBlockL->minUid); n += tPutI64(p ? p + n : p, pSstBlk->minUid);
n += tPutI64(p ? p + n : p, pBlockL->maxUid); n += tPutI64(p ? p + n : p, pSstBlk->maxUid);
n += tPutI64v(p ? p + n : p, pBlockL->minKey); n += tPutI64v(p ? p + n : p, pSstBlk->minKey);
n += tPutI64v(p ? p + n : p, pBlockL->maxKey); n += tPutI64v(p ? p + n : p, pSstBlk->maxKey);
n += tPutI64v(p ? p + n : p, pBlockL->minVer); n += tPutI64v(p ? p + n : p, pSstBlk->minVer);
n += tPutI64v(p ? p + n : p, pBlockL->maxVer); n += tPutI64v(p ? p + n : p, pSstBlk->maxVer);
n += tPutI32v(p ? p + n : p, pBlockL->nRow); n += tPutI32v(p ? p + n : p, pSstBlk->nRow);
n += tPutI64v(p ? p + n : p, pBlockL->bInfo.offset); n += tPutI64v(p ? p + n : p, pSstBlk->bInfo.offset);
n += tPutI32v(p ? p + n : p, pBlockL->bInfo.szBlock); n += tPutI32v(p ? p + n : p, pSstBlk->bInfo.szBlock);
n += tPutI32v(p ? p + n : p, pBlockL->bInfo.szKey); n += tPutI32v(p ? p + n : p, pSstBlk->bInfo.szKey);
return n; return n;
} }
int32_t tGetBlockL(uint8_t *p, void *ph) { int32_t tGetSstBlk(uint8_t *p, void *ph) {
int32_t n = 0; int32_t n = 0;
SBlockL *pBlockL = (SBlockL *)ph; SSstBlk *pSstBlk = (SSstBlk *)ph;
n += tGetI64(p + n, &pBlockL->suid); n += tGetI64(p + n, &pSstBlk->suid);
n += tGetI64(p + n, &pBlockL->minUid); n += tGetI64(p + n, &pSstBlk->minUid);
n += tGetI64(p + n, &pBlockL->maxUid); n += tGetI64(p + n, &pSstBlk->maxUid);
n += tGetI64v(p + n, &pBlockL->minKey); n += tGetI64v(p + n, &pSstBlk->minKey);
n += tGetI64v(p + n, &pBlockL->maxKey); n += tGetI64v(p + n, &pSstBlk->maxKey);
n += tGetI64v(p + n, &pBlockL->minVer); n += tGetI64v(p + n, &pSstBlk->minVer);
n += tGetI64v(p + n, &pBlockL->maxVer); n += tGetI64v(p + n, &pSstBlk->maxVer);
n += tGetI32v(p + n, &pBlockL->nRow); n += tGetI32v(p + n, &pSstBlk->nRow);
n += tGetI64v(p + n, &pBlockL->bInfo.offset); n += tGetI64v(p + n, &pSstBlk->bInfo.offset);
n += tGetI32v(p + n, &pBlockL->bInfo.szBlock); n += tGetI32v(p + n, &pSstBlk->bInfo.szBlock);
n += tGetI32v(p + n, &pBlockL->bInfo.szKey); n += tGetI32v(p + n, &pSstBlk->bInfo.szKey);
return n; return n;
} }