From ebd60a968c464cdf64cd392bb398366c15453107 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 1 Aug 2022 10:03:00 +0000 Subject: [PATCH 01/92] refact: tsdb last file optimize 1 --- source/dnode/vnode/src/inc/tsdb.h | 8 ++ source/dnode/vnode/src/tsdb/tsdbCommit.c | 75 +++++++++--- source/dnode/vnode/src/tsdb/tsdbFile.c | 2 + .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 109 ++++++++++++++++++ source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 50 ++++++++ 6 files changed, 231 insertions(+), 17 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index f1e980c026..8a6f473dc4 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -113,6 +113,9 @@ int32_t tPutBlock(uint8_t *p, void *ph); int32_t tGetBlock(uint8_t *p, void *ph); int32_t tBlockCmprFn(const void *p1, const void *p2); bool tBlockHasSma(SBlock *pBlock); +// SBlockL +int32_t tPutBlockL(uint8_t *p, void *ph); +int32_t tGetBlockL(uint8_t *p, void *ph); // SBlockIdx int32_t tPutBlockIdx(uint8_t *p, void *ph); int32_t tGetBlockIdx(uint8_t *p, void *ph); @@ -225,6 +228,7 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync); int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter); int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **ppBuf); int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, uint8_t **ppBuf, SBlockIdx *pBlockIdx); +int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL, uint8_t **ppBuf); int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2, SBlockIdx *pBlockIdx, SBlock *pBlock, int8_t cmprAlg); @@ -233,6 +237,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFReaderClose(SDataFReader **ppReader); int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppBuf); +int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf); int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData, uint8_t **ppBuf); int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int16_t *aColId, int32_t nCol, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); @@ -416,6 +421,7 @@ struct SBlock { }; struct SBlockL { + int64_t suid; struct { int64_t uid; int64_t version; @@ -452,6 +458,7 @@ struct SColData { struct SBlockData { int32_t nRow; + int64_t *aUid; int64_t *aVersion; TSKEY *aTSKEY; SArray *aIdx; // SArray @@ -513,6 +520,7 @@ struct SHeadFile { int64_t commitID; int64_t size; int64_t offset; + int64_t loffset; }; struct SDataFile { diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index e6db812865..dc1aa42dc6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -39,14 +39,18 @@ typedef struct { struct { SDataFReader *pReader; SArray *aBlockIdx; // SArray + SArray *aBlockL; // SArray SMapData mBlock; // SMapData, read from reader SBlockData bData; + SBlockData bDatal; } dReader; struct { SDataFWriter *pWriter; SArray *aBlockIdx; // SArray + SArray *aBlockL; // SArray SMapData mBlock; // SMapData SBlockData bData; + SBlockData bDatal; } dWriter; SSkmInfo skmTable; SSkmInfo skmRow; @@ -279,10 +283,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { // memory pCommitter->nextKey = TSKEY_MAX; - // old - taosArrayClear(pCommitter->dReader.aBlockIdx); - tMapDataReset(&pCommitter->dReader.mBlock); - tBlockDataReset(&pCommitter->dReader.bData); + // Reader pRSet = (SDFileSet *)taosArraySearch(pCommitter->fs.aDFileSet, &(SDFileSet){.fid = pCommitter->commitFid}, tDFileSetCmprFn, TD_EQ); if (pRSet) { @@ -291,22 +292,29 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { code = tsdbReadBlockIdx(pCommitter->dReader.pReader, pCommitter->dReader.aBlockIdx, NULL); if (code) goto _err; - } - // new + code = tsdbReadBlockL(pCommitter->dReader.pReader, pCommitter->dReader.aBlockL, NULL); + if (code) goto _err; + + } else { + pCommitter->dReader.pReader = NULL; + taosArrayClear(pCommitter->dReader.aBlockIdx); + taosArrayClear(pCommitter->dReader.aBlockL); + } + tMapDataReset(&pCommitter->dReader.mBlock); + tBlockDataReset(&pCommitter->dReader.bData); + tBlockDataReset(&pCommitter->dReader.bDatal); + + // Writer SHeadFile fHead; SDataFile fData; SLastFile fLast; SSmaFile fSma; SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .pLastF = &fLast, .pSmaF = &fSma}; - - taosArrayClear(pCommitter->dWriter.aBlockIdx); - tMapDataReset(&pCommitter->dWriter.mBlock); - tBlockDataReset(&pCommitter->dWriter.bData); if (pRSet) { wSet.diskId = pRSet->diskId; wSet.fid = pCommitter->commitFid; - fHead = (SHeadFile){.commitID = pCommitter->commitID, .offset = 0, .size = 0}; + fHead = (SHeadFile){.commitID = pCommitter->commitID, .offset = 0, .size = 0, .loffset = 0}; fData = *pRSet->pDataF; fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0}; fSma = *pRSet->pSmaF; @@ -319,7 +327,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { wSet.diskId = did; wSet.fid = pCommitter->commitFid; - fHead = (SHeadFile){.commitID = pCommitter->commitID, .offset = 0, .size = 0}; + fHead = (SHeadFile){.commitID = pCommitter->commitID, .offset = 0, .size = 0, .loffset = 0}; fData = (SDataFile){.commitID = pCommitter->commitID, .size = 0}; fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0}; fSma = (SSmaFile){.commitID = pCommitter->commitID, .size = 0}; @@ -327,6 +335,12 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet); if (code) goto _err; + taosArrayClear(pCommitter->dWriter.aBlockIdx); + taosArrayClear(pCommitter->dWriter.aBlockL); + tMapDataReset(&pCommitter->dWriter.mBlock); + tBlockDataReset(&pCommitter->dWriter.bData); + tBlockDataReset(&pCommitter->dWriter.bDatal); + _exit: return code; @@ -861,7 +875,11 @@ _err: static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { int32_t code = 0; - // write blockIdx + // write aBlockL + code = tsdbWriteBlockL(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockL, NULL); + if (code) goto _err; + + // write aBlockIdx code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx, NULL); if (code) goto _err; @@ -1001,14 +1019,15 @@ _err: static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { int32_t code = 0; + // Reader pCommitter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pCommitter->dReader.aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - pCommitter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); - if (pCommitter->dWriter.aBlockIdx == NULL) { + pCommitter->dReader.aBlockL = taosArrayInit(0, sizeof(SBlockL)); + if (pCommitter->dReader.aBlockL == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -1016,20 +1035,46 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { code = tBlockDataInit(&pCommitter->dReader.bData); if (code) goto _exit; + code = tBlockDataInit(&pCommitter->dReader.bDatal); + if (code) goto _exit; + + // Writer + pCommitter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); + if (pCommitter->dWriter.aBlockIdx == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + pCommitter->dWriter.aBlockL = taosArrayInit(0, sizeof(SBlockL)); + if (pCommitter->dWriter.aBlockL == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + code = tBlockDataInit(&pCommitter->dWriter.bData); if (code) goto _exit; + code = tBlockDataInit(&pCommitter->dWriter.bDatal); + if (code) goto _exit; + _exit: return code; } static void tsdbCommitDataEnd(SCommitter *pCommitter) { + // Reader taosArrayDestroy(pCommitter->dReader.aBlockIdx); + taosArrayDestroy(pCommitter->dReader.aBlockL); tMapDataClear(&pCommitter->dReader.mBlock); tBlockDataClear(&pCommitter->dReader.bData, 1); + tBlockDataClear(&pCommitter->dReader.bDatal, 1); + + // Writer taosArrayDestroy(pCommitter->dWriter.aBlockIdx); + taosArrayDestroy(pCommitter->dWriter.aBlockL); tMapDataClear(&pCommitter->dWriter.mBlock); tBlockDataClear(&pCommitter->dWriter.bData, 1); + tBlockDataClear(&pCommitter->dWriter.bDatal, 1); tTSchemaDestroy(pCommitter->skmTable.pTSchema); tTSchemaDestroy(pCommitter->skmRow.pTSchema); } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 135ee23d44..f9f2c0df6b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -21,6 +21,7 @@ int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile) { n += tPutI64v(p ? p + n : p, pHeadFile->commitID); n += tPutI64v(p ? p + n : p, pHeadFile->size); n += tPutI64v(p ? p + n : p, pHeadFile->offset); + n += tPutI64v(p ? p + n : p, pHeadFile->loffset); return n; } @@ -31,6 +32,7 @@ static int32_t tGetHeadFile(uint8_t *p, SHeadFile *pHeadFile) { n += tGetI64v(p + n, &pHeadFile->commitID); n += tGetI64v(p + n, &pHeadFile->size); n += tGetI64v(p + n, &pHeadFile->offset); + n += tGetI64v(p + n, &pHeadFile->loffset); return n; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 7365ac23b8..f33bb9e455 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -597,6 +597,68 @@ _err: return code; } +int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf) { + int32_t code = 0; + int64_t offset = pReader->pSet->pHeadF->loffset; + int64_t size = pReader->pSet->pHeadF->offset - offset; + int64_t n; + uint32_t delimiter; + uint8_t *pBuf = NULL; + SBlockL blockl; + + if (!ppBuf) ppBuf = &pBuf; + + // alloc + code = tRealloc(ppBuf, size); + if (code) goto _err; + + // seek + if (taosLSeekFile(pReader->pHeadFD, offset, SEEK_SET) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // read + n = taosReadFile(pReader->pHeadFD, *ppBuf, size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } else if (n < size) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + + // check + if (!taosCheckChecksumWhole(*ppBuf, size)) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + + // decode + n = 0; + n = tGetU32(*ppBuf + n, &delimiter); + ASSERT(delimiter == TSDB_FILE_DLMT); + + taosArrayClear(aBlockL); + while (n < size - sizeof(TSCKSUM)) { + n += tGetBlockL(*ppBuf + n, &blockl); + + if (taosArrayPush(aBlockL, &blockl) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + } + + ASSERT(n + sizeof(TSCKSUM) == size); + + tFree(pBuf); + return code; + +_err: + tsdbError("vgId:%d read blockl failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + return code; +} + int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBlock, uint8_t **ppBuf) { int32_t code = 0; int64_t offset = pBlockIdx->offset; @@ -1593,6 +1655,53 @@ _err: return code; } +int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL, uint8_t **ppBuf) { + int32_t code = 0; + SHeadFile *pHeadFile = &pWriter->fHead; + uint8_t *pBuf = NULL; + int64_t size; + int64_t n; + + // size + size = sizeof(uint32_t); + for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aBlockL); iBlockL++) { + size += tPutBlockL(NULL, taosArrayGet(aBlockL, iBlockL)); + } + size += sizeof(TSCKSUM); + + // alloc + if (!ppBuf) ppBuf = &pBuf; + code = tRealloc(ppBuf, size); + if (code) goto _err; + + // encode + n = 0; + n += tPutU32(*ppBuf + n, TSDB_FILE_DLMT); + for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aBlockL); iBlockL++) { + n += tPutBlockL(*ppBuf + n, taosArrayGet(aBlockL, iBlockL)); + } + taosCalcChecksumAppend(0, *ppBuf, size); + + ASSERT(n + sizeof(TSCKSUM) == size); + + // write + n = taosWriteFile(pWriter->pHeadFD, *ppBuf, size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // update + pHeadFile->loffset = pHeadFile->size; + pHeadFile->size += size; + + return code; + +_err: + tsdbError("vgId:%d tsdb write blockl failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + return code; +} + static void tsdbUpdateBlockInfo(SBlockData *pBlockData, SBlock *pBlock) { for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { TSDBKEY key = {.ts = pBlockData->aTSKEY[iRow], .version = pBlockData->aVersion[iRow]}; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 6bb2b8c253..750a5dd164 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -911,14 +911,14 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 if (pSet) { wSet.diskId = pSet->diskId; wSet.fid = fid; - fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0}; + fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0, .loffset = 0}; fData = *pSet->pDataF; fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0}; fSma = *pSet->pSmaF; } else { wSet.diskId = (SDiskID){.level = 0, .id = 0}; wSet.fid = fid; - fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0}; + fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0, .loffset = 0}; fData = (SDataFile){.commitID = pWriter->commitID, .size = 0}; fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0}; fSma = (SSmaFile){.commitID = pWriter->commitID, .size = 0}; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index d612e9bb10..4e0d1b2402 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -295,6 +295,56 @@ bool tBlockHasSma(SBlock *pBlock) { return pBlock->aSubBlock[0].nSma > 0; } +// SBlockL ====================================================== +int32_t tPutBlockL(uint8_t *p, void *ph) { + int32_t n = 0; + SBlockL *pBlockL = (SBlockL *)ph; + + n += tPutI64(p ? p + n : p, pBlockL->suid); + n += tPutI64(p ? p + n : p, pBlockL->minKey.uid); + n += tPutI64v(p ? p + n : p, pBlockL->minKey.version); + n += tPutI64(p ? p + n : p, pBlockL->minKey.ts); + n += tPutI64(p ? p + n : p, pBlockL->maxKey.uid); + n += tPutI64v(p ? p + n : p, pBlockL->maxKey.version); + n += tPutI64(p ? p + n : p, pBlockL->maxKey.ts); + n += tPutI64v(p ? p + n : p, pBlockL->minVer); + n += tPutI64v(p ? p + n : p, pBlockL->maxVer); + n += tPutI32v(p ? p + n : p, pBlockL->nRow); + n += tPutI8(p ? p + n : p, pBlockL->cmprAlg); + n += tPutI64v(p ? p + n : p, pBlockL->offset); + n += tPutI32v(p ? p + n : p, pBlockL->szBlock); + n += tPutI32v(p ? p + n : p, pBlockL->szBlockCol); + n += tPutI32v(p ? p + n : p, pBlockL->szUid); + n += tPutI32v(p ? p + n : p, pBlockL->szVer); + n += tPutI32v(p ? p + n : p, pBlockL->szTSKEY); + + return n; +} + +int32_t tGetBlockL(uint8_t *p, void *ph) { + int32_t n = 0; + SBlockL *pBlockL = (SBlockL *)ph; + + n += tGetI64(p + n, &pBlockL->suid); + n += tGetI64(p + n, &pBlockL->minKey.uid); + n += tGetI64v(p + n, &pBlockL->minKey.version); + n += tGetI64(p + n, &pBlockL->minKey.ts); + n += tGetI64(p + n, &pBlockL->maxKey.uid); + n += tGetI64v(p + n, &pBlockL->maxKey.version); + n += tGetI64(p + n, &pBlockL->maxKey.ts); + n += tGetI64v(p + n, &pBlockL->minVer); + n += tGetI64v(p + n, &pBlockL->maxVer); + n += tGetI32v(p + n, &pBlockL->nRow); + n += tGetI8(p + n, &pBlockL->cmprAlg); + n += tGetI64v(p + n, &pBlockL->offset); + n += tGetI32v(p + n, &pBlockL->szBlock); + n += tGetI32v(p + n, &pBlockL->szBlockCol); + n += tGetI32v(p + n, &pBlockL->szUid); + n += tGetI32v(p + n, &pBlockL->szVer); + n += tGetI32v(p + n, &pBlockL->szTSKEY); + + return n; +} // SBlockCol ====================================================== int32_t tPutBlockCol(uint8_t *p, void *ph) { From 2d0882edba25b2e87491a0be4fe2f21df3536382 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 2 Aug 2022 09:06:18 +0000 Subject: [PATCH 02/92] more last file refact --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 226 +++++++++++++++++------ 1 file changed, 165 insertions(+), 61 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 3900adf205..2f57992280 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -20,6 +20,12 @@ typedef struct { STSchema *pTSchema; } SSkmInfo; +typedef struct { + int64_t suid; + int64_t uid; + TSDBROW row; +} SRowInfo; + typedef struct { STsdb *pTsdb; /* commit data */ @@ -38,11 +44,20 @@ typedef struct { // commit file data struct { SDataFReader *pReader; - SArray *aBlockIdx; // SArray - SArray *aBlockL; // SArray - SMapData mBlock; // SMapData, read from reader - SBlockData bData; - SBlockData bDatal; + // data + SArray *aBlockIdx; // SArray + int32_t iBlockIdx; + SBlockIdx *pBlockIdx; + SMapData mBlock; // SMapData + SBlockData bData; + // last + SArray *aBlockL; // SArray + int32_t iBlockL; + SBlockL *pBlockL; + SBlockData bDatal; + int32_t iRow; + SRowInfo *pRow; + SRowInfo row; } dReader; struct { SDataFWriter *pWriter; @@ -290,20 +305,46 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { code = tsdbDataFReaderOpen(&pCommitter->dReader.pReader, pTsdb, pRSet); if (code) goto _err; + // data code = tsdbReadBlockIdx(pCommitter->dReader.pReader, pCommitter->dReader.aBlockIdx, NULL); if (code) goto _err; + pCommitter->dReader.iBlockIdx = 0; + if (pCommitter->dReader.iBlockIdx < taosArrayGetSize(pCommitter->dReader.aBlockIdx)) { + pCommitter->dReader.pBlockIdx = + (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx); + + code = + tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock, NULL); + if (code) goto _err; + } else { + pCommitter->dReader.pBlockIdx = NULL; + } + + // last code = tsdbReadBlockL(pCommitter->dReader.pReader, pCommitter->dReader.aBlockL, NULL); if (code) goto _err; + pCommitter->dReader.iBlockL = 0; + if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { + pCommitter->dReader.pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); + + // TODO: code = tsdbReadBlockData(pCommitter->dReader.pReader, NULL, pBlockL, &pCommitter->dReader.bDatal, NULL, + // NULL); + if (code) goto _err; + + pCommitter->dReader.iRow = 0; + pCommitter->dReader.pRow = &pCommitter->dReader.row; + pCommitter->dReader.pRow->suid = pCommitter->dReader.pBlockL->suid; + pCommitter->dReader.pRow->uid = pCommitter->dReader.bDatal.aUid[pCommitter->dReader.iRow]; + pCommitter->dReader.pRow->row = tsdbRowFromBlockData(&pCommitter->dReader.bDatal, pCommitter->dReader.iRow); + } else { + pCommitter->dReader.pRow = NULL; + } } else { - pCommitter->dReader.pReader = NULL; - taosArrayClear(pCommitter->dReader.aBlockIdx); - taosArrayClear(pCommitter->dReader.aBlockL); + pCommitter->dReader.pBlockIdx = NULL; + pCommitter->dReader.pRow = NULL; } - tMapDataReset(&pCommitter->dReader.mBlock); - tBlockDataReset(&pCommitter->dReader.bData); - tBlockDataReset(&pCommitter->dReader.bDatal); // Writer SHeadFile fHead; @@ -707,7 +748,7 @@ _err: return code; } -static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData, SBlockIdx *pBlockIdx) { +static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { int32_t code = 0; STbDataIter iter = {0}; STbDataIter *pIter = &iter; @@ -716,6 +757,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData, SBl int32_t nBlock; int64_t suid; int64_t uid; + SBlockIdx *pBlockIdx = NULL; if (pTbData) { tsdbTbDataIterOpen(pTbData, &(TSDBKEY){.ts = pCommitter->minKey, .version = VERSION_MIN}, 0, pIter); @@ -908,6 +950,95 @@ _err: return code; } +static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { + int32_t code = 0; + + // data + while (true) { + if (pCommitter->dReader.pBlockIdx == NULL || tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) >= 0) break; + + SBlockIdx blockIdx = *pCommitter->dReader.pBlockIdx; + code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, NULL, &blockIdx); + if (code) goto _err; + + if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + pCommitter->dReader.iBlockIdx++; + if (pCommitter->dReader.iBlockIdx < taosArrayGetSize(pCommitter->dReader.aBlockIdx)) { + pCommitter->dReader.pBlockIdx = + (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx); + + code = + tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock, NULL); + if (code) goto _err; + } else { + pCommitter->dReader.pBlockIdx = NULL; + } + } + + // last + SBlockL blockL; + while (true) { + if (pCommitter->dReader.pRow == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRow, &toTable) >= 0) break; + + // check if same suid + if (0) { + goto _write_block_data; + } + + // append + code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &pCommitter->dReader.pRow->row, NULL); + if (code) goto _err; + + // next + pCommitter->dReader.iRow++; + if (pCommitter->dReader.iRow < pCommitter->dReader.bDatal.nRow) { + pCommitter->dReader.pRow->uid = pCommitter->dReader.bDatal.aUid[pCommitter->dReader.iRow]; + pCommitter->dReader.pRow->row = tsdbRowFromBlockData(&pCommitter->dReader.bDatal, pCommitter->dReader.iRow); + } else { + pCommitter->dReader.iBlockL++; + if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { + pCommitter->dReader.pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); + + // TODO: code = tsdbReadBlockData(pCommitter->dReader.pReader, NULL, pBlockL, &pCommitter->dReader.bDatal, NULL, + // NULL); + if (code) goto _err; + + pCommitter->dReader.iRow = 0; + pCommitter->dReader.pRow->suid = pCommitter->dReader.pBlockL->suid; + pCommitter->dReader.pRow->uid = pCommitter->dReader.bDatal.aUid[pCommitter->dReader.iRow]; + pCommitter->dReader.pRow->row = tsdbRowFromBlockData(&pCommitter->dReader.bDatal, pCommitter->dReader.iRow); + } else { + pCommitter->dReader.pRow = NULL; + } + } + + // write + if (pCommitter->dWriter.bDatal.nRow < pCommitter->maxRow) continue; + + _write_block_data: + code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, NULL, NULL, NULL, NULL, + pCommitter->cmprAlg); // todo + if (code) goto _err; + + if (taosArrayPush(pCommitter->dWriter.aBlockL, &blockL) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + tBlockDataClearData(&pCommitter->dWriter.bDatal); + } + + return code; + +_err: + tsdbError("vgId:%d tsdb move commit data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + return code; +} + static int32_t tsdbCommitFileData(SCommitter *pCommitter) { int32_t code = 0; STsdb *pTsdb = pCommitter->pTsdb; @@ -918,59 +1049,32 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) { if (code) goto _err; // commit file data impl - int32_t iTbData = 0; - int32_t nTbData = taosArrayGetSize(pMemTable->aTbData); - int32_t iBlockIdx = 0; - int32_t nBlockIdx = taosArrayGetSize(pCommitter->dReader.aBlockIdx); - STbData *pTbData; - SBlockIdx *pBlockIdx; + for (int32_t iTbData = 0; iTbData < taosArrayGetSize(pMemTable->aTbData); iTbData++) { + STbData *pTbData = (STbData *)taosArrayGetP(pMemTable->aTbData, iTbData); - ASSERT(nTbData > 0); + // move commit until current (suid, uid) + code = tsdbMoveCommitData(pCommitter, (TABLEID){.suid = pTbData->suid, .uid = pTbData->uid}); + if (code) goto _err; - pTbData = (STbData *)taosArrayGetP(pMemTable->aTbData, iTbData); - pBlockIdx = (iBlockIdx < nBlockIdx) ? (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, iBlockIdx) : NULL; - while (pTbData || pBlockIdx) { - if (pTbData && pBlockIdx) { - int32_t c = tTABLEIDCmprFn(pTbData, pBlockIdx); + // commit current table data commit + code = tsdbCommitTableData(pCommitter, pTbData); + if (code) goto _err; + } - if (c == 0) { - goto _commit_table_mem_and_disk; - } else if (c < 0) { - goto _commit_table_mem_data; - } else { - goto _commit_table_disk_data; - } - } else if (pBlockIdx) { - goto _commit_table_disk_data; - } else { - goto _commit_table_mem_data; + code = tsdbMoveCommitData(pCommitter, (TABLEID){.suid = INT64_MAX, .uid = INT64_MAX}); + if (code) goto _err; + + if (pCommitter->dWriter.bDatal.nRow > 0) { + SBlockL blockL; + + code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, NULL, NULL, NULL, NULL, + pCommitter->cmprAlg); + if (code) goto _err; + + if (taosArrayPush(pCommitter->dWriter.aBlockL, &blockL) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; } - - _commit_table_mem_data: - code = tsdbCommitTableData(pCommitter, pTbData, NULL); - if (code) goto _err; - - iTbData++; - pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pMemTable->aTbData, iTbData) : NULL; - continue; - - _commit_table_disk_data: - code = tsdbCommitTableData(pCommitter, NULL, pBlockIdx); - if (code) goto _err; - - iBlockIdx++; - pBlockIdx = (iBlockIdx < nBlockIdx) ? (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, iBlockIdx) : NULL; - continue; - - _commit_table_mem_and_disk: - code = tsdbCommitTableData(pCommitter, pTbData, pBlockIdx); - if (code) goto _err; - - iBlockIdx++; - pBlockIdx = (iBlockIdx < nBlockIdx) ? (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, iBlockIdx) : NULL; - iTbData++; - pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pMemTable->aTbData, iTbData) : NULL; - continue; } // commit file data end From f24317fbfff1fefc930d91db5a222e31f2000a76 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 3 Aug 2022 01:45:56 +0000 Subject: [PATCH 03/92] more last file refact --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 211 ++++++++++++++++++----- 1 file changed, 171 insertions(+), 40 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 2f57992280..704aa0aab0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -569,7 +569,8 @@ _err: } static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter, TSDBKEY toKey, int8_t toDataOnly) { - int32_t code = 0; + int32_t code = 0; +#if 0 TSDBROW *pRow; SBlock block; SBlock *pBlock = █ @@ -623,6 +624,7 @@ static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter tBlockDataClearData(pBlockData); } +#endif return code; _err: @@ -653,25 +655,25 @@ _err: return code; } -static int32_t tsdbCommitTableDataEnd(SCommitter *pCommitter, int64_t suid, int64_t uid) { - int32_t code = 0; - SBlockIdx blockIdx = {.suid = suid, .uid = uid}; - SBlockIdx *pBlockIdx = &blockIdx; +// static int32_t tsdbCommitTableDataEnd(SCommitter *pCommitter, int64_t suid, int64_t uid) { +// int32_t code = 0; +// SBlockIdx blockIdx = {.suid = suid, .uid = uid}; +// SBlockIdx *pBlockIdx = &blockIdx; - code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, NULL, pBlockIdx); - if (code) goto _err; +// code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, NULL, pBlockIdx); +// if (code) goto _err; - if (taosArrayPush(pCommitter->dWriter.aBlockIdx, pBlockIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } +// if (taosArrayPush(pCommitter->dWriter.aBlockIdx, pBlockIdx) == NULL) { +// code = TSDB_CODE_OUT_OF_MEMORY; +// goto _err; +// } - return code; +// return code; -_err: - tsdbError("vgId:%d, commit table data end failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); - return code; -} +// _err: +// tsdbError("vgId:%d, commit table data end failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); +// return code; +// } static int32_t tsdbGetOvlpNRow(STbDataIter *pIter, SBlock *pBlock) { int32_t nRow = 0; @@ -748,6 +750,145 @@ _err: return code; } +static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { + int32_t code = 0; + + ASSERT(pCommitter->dReader.pBlockIdx == NULL || tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, pTbData) >= 0); + ASSERT(pCommitter->dReader.pRow == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRow, pTbData) >= 0); + + // end last if need + if (pTbData->suid == 0 || pTbData->suid != 0 /*todo*/) { + if (pCommitter->dWriter.bDatal.nRow > 0) { + // TODO: code = tsdbCommitBlockDataL(pCommitter); + if (code) goto _err; + } + } + + // merge commit table data + STbDataIter iter = {0}; + STbDataIter *pIter = &iter; + TSDBROW *pRow; + + tsdbTbDataIterOpen(pTbData, &(TSDBKEY){.ts = pCommitter->minKey, .version = VERSION_MIN}, 0, pIter); + pRow = tsdbTbDataIterGet(pIter); + if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { + pRow = NULL; + } + + if (pRow == NULL) goto _exit; + + SBlockIdx *pBlockIdx = NULL; + int32_t iBlock = 0; + SBlock block; + SBlock *pBlock = █ + + if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pBlockIdx) == 0) { + pBlockIdx = pCommitter->dReader.pBlockIdx; + } + + if (pBlockIdx && iBlock < pCommitter->dReader.mBlock.nItem) { + tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); + } else { + pBlock = NULL; + } + + tMapDataReset(&pCommitter->dWriter.mBlock); + while (pBlock && pRow) { + int32_t c = tBlockCmprFn(pBlock, &(SBlock){.minKey = TSDBROW_KEY(pRow), .maxKey = TSDBROW_KEY(pRow)}); + if (c < 0) { // disk + code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock); + if (code) goto _err; + + // next + iBlock++; + if (iBlock < pCommitter->dReader.mBlock.nItem) { + tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); + } else { + pBlock = NULL; + } + } else if (c < 0) { // memory + code = tsdbCommitTableMemData(pCommitter, pIter, pBlock->minKey, 1); + if (code) goto _err; + + // next + pRow = tsdbTbDataIterGet(pIter); + if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { + pRow = NULL; + } + } else { // merge + int32_t nOvlp = tsdbGetOvlpNRow(pIter, pBlock); + + ASSERT(nOvlp > 0); + + if (pBlock->nRow + nOvlp <= pCommitter->maxRow && pBlock->nSubBlock < TSDB_MAX_SUBBLOCKS) { + code = tsdbMergeAsSubBlock(pCommitter, pIter, pBlock); + if (code) goto _err; + } else { + // code = tsdbMergeTableData(pCommitter, pIter, pBlock, NULL, 1); + if (code) goto _err; + } + + // next + pRow = tsdbTbDataIterGet(pIter); + if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { + pRow = NULL; + } + iBlock++; + if (iBlock < pCommitter->dReader.mBlock.nItem) { + tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); + } else { + pBlock = NULL; + } + } + } + + while (pBlock) { + code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock); + if (code) goto _err; + + iBlock++; + if (iBlock < pCommitter->dReader.mBlock.nItem) { + tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); + } else { + pBlock = NULL; + } + } + + if (pRow) { + code = + tsdbCommitTableMemData(pCommitter, pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = VERSION_MIN}, 0); + if (code) goto _err; + + pRow = tsdbTbDataIterGet(pIter); + ASSERT(pRow == NULL || TSDBROW_TS(pRow) > pCommitter->maxKey); + } + + // end + if (pCommitter->dWriter.mBlock.nItem > 0) { + SBlockIdx blockIdx = {.suid = pTbData->suid, .uid = pTbData->uid}; + code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, NULL, &blockIdx); + if (code) goto _err; + + if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + } + +_exit: + pRow = tsdbTbDataIterGet(pIter); + if (pRow) { + pCommitter->nextKey = TMIN(pCommitter->nextKey, TSDBROW_TS(pRow)); + } + + return code; + +_err: + tsdbError("vgId:%d tsdb commit table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + return code; +} + +#if 0 static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { int32_t code = 0; STbDataIter iter = {0}; @@ -913,6 +1054,7 @@ _err: tsdbError("vgId:%d, tsdb commit table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); return code; } +#endif static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { int32_t code = 0; @@ -985,8 +1127,14 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { if (pCommitter->dReader.pRow == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRow, &toTable) >= 0) break; // check if same suid - if (0) { - goto _write_block_data; + if (pCommitter->dReader.pRow->suid == 0) { + if (pCommitter->dReader.pRow->uid != 0 /*todo*/) { + // code = tsdbCommitBlockDataL(pCommitter); + if (code) goto _err; + } + } else if (pCommitter->dReader.pRow->suid != 0 /*todo*/) { + // code = tsdbCommitBlockDataL(pCommitter); + if (code) goto _err; } // append @@ -1017,19 +1165,10 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { } // write - if (pCommitter->dWriter.bDatal.nRow < pCommitter->maxRow) continue; - - _write_block_data: - code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, NULL, NULL, NULL, NULL, - pCommitter->cmprAlg); // todo - if (code) goto _err; - - if (taosArrayPush(pCommitter->dWriter.aBlockL, &blockL) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + if (pCommitter->dWriter.bDatal.nRow >= pCommitter->maxRow) { + // code = tsdbCommitBlockDataL(pCommitter); + if (code) goto _err; } - - tBlockDataClearData(&pCommitter->dWriter.bDatal); } return code; @@ -1056,7 +1195,7 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) { code = tsdbMoveCommitData(pCommitter, (TABLEID){.suid = pTbData->suid, .uid = pTbData->uid}); if (code) goto _err; - // commit current table data commit + // commit current table data code = tsdbCommitTableData(pCommitter, pTbData); if (code) goto _err; } @@ -1065,16 +1204,8 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) { if (code) goto _err; if (pCommitter->dWriter.bDatal.nRow > 0) { - SBlockL blockL; - - code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, NULL, NULL, NULL, NULL, - pCommitter->cmprAlg); + // code = tsdbCommitBlockDataL(pCommitter); if (code) goto _err; - - if (taosArrayPush(pCommitter->dWriter.aBlockL, &blockL) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } } // commit file data end From e827d0bb7a5e58643f38bd21b6a28a2a5ff28c90 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 3 Aug 2022 08:50:36 +0000 Subject: [PATCH 04/92] more last file refact --- source/dnode/vnode/src/inc/tsdb.h | 13 + source/dnode/vnode/src/tsdb/tsdbCommit.c | 500 +++++++++--------- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 34 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 2 + 4 files changed, 289 insertions(+), 260 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 8a6f473dc4..af256c0e19 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -233,6 +233,12 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_ SBlockIdx *pBlockIdx, SBlock *pBlock, int8_t cmprAlg); int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); + +/* new */ +int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock *pBlock, uint8_t **ppBuf1, + uint8_t **ppBuf2, int8_t cmprAlg); +int32_t tsdbWriteLastBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockL *pBlockL, uint8_t **ppBuf1, + uint8_t **ppBuf2, int8_t cmprAlg); // SDataFReader int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFReaderClose(SDataFReader **ppReader); @@ -244,6 +250,12 @@ int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBl int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg, uint8_t **ppBuf); + +/* new */ +int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, uint8_t **ppBuf1, + uint8_t **ppBuf2); +int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, uint8_t **ppBuf1, + uint8_t **ppBuf2); // SDelFWriter int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync); @@ -457,6 +469,7 @@ struct SColData { }; struct SBlockData { + int64_t suid; int32_t nRow; int64_t *aUid; int64_t *aVersion; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 704aa0aab0..70892acb0a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -56,8 +56,8 @@ typedef struct { SBlockL *pBlockL; SBlockData bDatal; int32_t iRow; - SRowInfo *pRow; - SRowInfo row; + SRowInfo *pRowInfo; + SRowInfo rowInfo; } dReader; struct { SDataFWriter *pWriter; @@ -290,12 +290,46 @@ _err: return code; } +static int32_t tsdbCommitNextLastRow(SCommitter *pCommitter) { + int32_t code = 0; + + ASSERT(pCommitter->dReader.pReader); + ASSERT(pCommitter->dReader.pRowInfo); + + pCommitter->dReader.iRow++; + if (pCommitter->dReader.iRow < pCommitter->dReader.bDatal.nRow) { + pCommitter->dReader.pRowInfo->uid = pCommitter->dReader.bData.aUid[pCommitter->dReader.iRow]; + pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(&pCommitter->dReader.bDatal, pCommitter->dReader.iRow); + } else { + pCommitter->dReader.iBlockL++; + if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { + pCommitter->dReader.pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); + code = tsdbReadLastBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockL, &pCommitter->dReader.bDatal, + NULL, NULL); + if (code) goto _exit; + + pCommitter->dReader.iRow = 0; + pCommitter->dReader.pRowInfo->suid = pCommitter->dReader.pBlockL->suid; + pCommitter->dReader.pRowInfo->uid = pCommitter->dReader.bData.aUid[pCommitter->dReader.iRow]; + pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(&pCommitter->dReader.bDatal, pCommitter->dReader.iRow); + } else { + pCommitter->dReader.pRowInfo = NULL; + } + } + +_exit: + return code; +} + static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { int32_t code = 0; STsdb *pTsdb = pCommitter->pTsdb; SDFileSet *pRSet = NULL; // memory + pCommitter->commitFid = tsdbKeyFid(pCommitter->nextKey, pCommitter->minutes, pCommitter->precision); + tsdbFidKeyRange(pCommitter->commitFid, pCommitter->minutes, pCommitter->precision, &pCommitter->minKey, + &pCommitter->maxKey); pCommitter->nextKey = TSKEY_MAX; // Reader @@ -325,25 +359,15 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { code = tsdbReadBlockL(pCommitter->dReader.pReader, pCommitter->dReader.aBlockL, NULL); if (code) goto _err; - pCommitter->dReader.iBlockL = 0; - if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { - pCommitter->dReader.pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); - - // TODO: code = tsdbReadBlockData(pCommitter->dReader.pReader, NULL, pBlockL, &pCommitter->dReader.bDatal, NULL, - // NULL); - if (code) goto _err; - - pCommitter->dReader.iRow = 0; - pCommitter->dReader.pRow = &pCommitter->dReader.row; - pCommitter->dReader.pRow->suid = pCommitter->dReader.pBlockL->suid; - pCommitter->dReader.pRow->uid = pCommitter->dReader.bDatal.aUid[pCommitter->dReader.iRow]; - pCommitter->dReader.pRow->row = tsdbRowFromBlockData(&pCommitter->dReader.bDatal, pCommitter->dReader.iRow); - } else { - pCommitter->dReader.pRow = NULL; - } + pCommitter->dReader.iBlockL = -1; + pCommitter->dReader.bDatal.nRow = 0; + pCommitter->dReader.iRow = -1; + pCommitter->dReader.pRowInfo = &pCommitter->dReader.rowInfo; + code = tsdbCommitNextLastRow(pCommitter); + if (code) goto _err; } else { pCommitter->dReader.pBlockIdx = NULL; - pCommitter->dReader.pRow = NULL; + pCommitter->dReader.pRowInfo = NULL; } // Writer @@ -463,6 +487,46 @@ _err: return code; } +static int32_t tsdbCommitDataBlock(SCommitter *pCommitter) { + int32_t code = 0; + SBlock block; + + ASSERT(pCommitter->dWriter.bData.nRow > 0); + + code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bData, &block, NULL, NULL, + pCommitter->cmprAlg); + if (code) goto _exit; + + code = tMapDataPutItem(&pCommitter->dWriter.mBlock, &block, tPutBlock); + if (code) goto _exit; + + tBlockDataClearData(&pCommitter->dWriter.bData); + +_exit: + return code; +} + +static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { + int32_t code = 0; + SBlockL blockL; + + ASSERT(pCommitter->dWriter.bDatal.nRow > 0); + + code = tsdbWriteLastBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, &blockL, NULL, NULL, + pCommitter->cmprAlg); + if (code) goto _exit; + + if (taosArrayPush(pCommitter->dWriter.aBlockL, &blockL) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + tBlockDataClearData(&pCommitter->dWriter.bDatal); + +_exit: + return code; +} + static int32_t tsdbMergeTableData(SCommitter *pCommitter, STbDataIter *pIter, SBlock *pBlockMerge, TSDBKEY toKey, int8_t toDataOnly) { int32_t code = 0; @@ -675,22 +739,16 @@ _err: // return code; // } -static int32_t tsdbGetOvlpNRow(STbDataIter *pIter, SBlock *pBlock) { - int32_t nRow = 0; - TSDBROW *pRow; - TSDBKEY key; - int32_t c = 0; +static int32_t tsdbGetNumOfRowsLessThan(STbDataIter *pIter, TSDBKEY key) { + int32_t nRow = 0; + STbDataIter iter = *pIter; - - iter.pRow = NULL; while (true) { - pRow = tsdbTbDataIterGet(&iter); - + TSDBROW *pRow = tsdbTbDataIterGet(&iter); if (pRow == NULL) break; - key = TSDBROW_KEY(pRow); - c = tBlockCmprFn(&(SBlock){.maxKey = key, .minKey = key}, pBlock); - if (c == 0) { + int32_t c = tsdbKeyCmprFn(&TSDBROW_KEY(pRow), &key); + if (c < 0) { nRow++; tsdbTbDataIterNext(&iter); } else if (c > 0) { @@ -750,11 +808,106 @@ _err: return code; } +static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter, int32_t *nRow) { + int32_t code = 0; + STbData *pTbData = pIter->pTbData; + TSDBROW *pRow = tsdbTbDataIterGet(pIter); + SRowInfo *pRowInfo = pCommitter->dReader.pRowInfo; + + while (pRow && pRowInfo) { + int32_t c = tsdbRowCmprFn(pRow, &pRowInfo->row); + if (c < 0) { + code = tBlockDataAppendRow(&pCommitter->dWriter.bData, pRow, NULL); + if (code) goto _err; + + tsdbTbDataIterNext(pIter); + pRow = tsdbTbDataIterGet(pIter); + if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { + pRow = NULL; + } + } else if (c > 0) { + code = tBlockDataAppendRow(&pCommitter->dWriter.bData, &pRowInfo->row, NULL); + if (code) goto _err; + + pCommitter->dReader.iRow++; + if (pCommitter->dReader.iRow < pCommitter->dReader.bDatal.nRow) { + // todo + } else { + pCommitter->dReader.iBlockL++; + if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { + // todo + } else { + // todo + } + } + + pRowInfo = pCommitter->dReader.pRowInfo; + if (pRowInfo && (pRowInfo->suid != pTbData->suid || pRowInfo->uid != pTbData->uid)) { + pRowInfo = NULL; + } + } else { + ASSERT(0); + } + + (*nRow)--; + ASSERT(*nRow >= 0); + if (pCommitter->dWriter.bData.nRow >= pCommitter->maxRow * 4 / 5) goto _write_block_data; + } + + while (pRow) { + code = tBlockDataAppendRow(&pCommitter->dWriter.bData, pRow, NULL); + if (code) goto _err; + + tsdbTbDataIterNext(pIter); + pRow = tsdbTbDataIterGet(pIter); + if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { + pRow = NULL; + } + + (*nRow)--; + ASSERT(*nRow >= 0); + if (pCommitter->dWriter.bData.nRow >= pCommitter->maxRow * 4 / 5) goto _write_block_data; + } + + while (pRowInfo) { + code = tBlockDataAppendRow(&pCommitter->dWriter.bData, &pRowInfo->row, NULL); + if (code) goto _err; + + pCommitter->dReader.iRow++; + if (pCommitter->dReader.iRow < pCommitter->dReader.bDatal.nRow) { + // todo + } else { + pCommitter->dReader.iBlockL++; + if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { + // todo + } else { + // todo + } + } + + pRowInfo = pCommitter->dReader.pRowInfo; + if (pRowInfo && (pRowInfo->suid != pTbData->suid || pRowInfo->uid != pTbData->uid)) { + pRowInfo = NULL; + } + + (*nRow)--; + ASSERT(*nRow >= 0); + if (pCommitter->dWriter.bData.nRow >= pCommitter->maxRow * 4 / 5) goto _write_block_data; + } + + SBlock block; +_write_block_data: + return code; + +_err: + return code; +} + static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { int32_t code = 0; ASSERT(pCommitter->dReader.pBlockIdx == NULL || tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, pTbData) >= 0); - ASSERT(pCommitter->dReader.pRow == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRow, pTbData) >= 0); + ASSERT(pCommitter->dReader.pRowInfo == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRowInfo, pTbData) >= 0); // end last if need if (pTbData->suid == 0 || pTbData->suid != 0 /*todo*/) { @@ -816,7 +969,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { pRow = NULL; } } else { // merge - int32_t nOvlp = tsdbGetOvlpNRow(pIter, pBlock); + int32_t nOvlp = tsdbGetNumOfRowsLessThan(pIter, pBlock->maxKey); ASSERT(nOvlp > 0); @@ -854,13 +1007,20 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { } } - if (pRow) { - code = - tsdbCommitTableMemData(pCommitter, pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = VERSION_MIN}, 0); - if (code) goto _err; + // merge with last + int32_t nRowLeft = tsdbGetNumOfRowsLessThan(pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = VERSION_MIN}); + if (pCommitter->dReader.pRowInfo) { + for (int32_t iRow = pCommitter->dReader.iRow; iRow < pCommitter->dReader.bDatal.nRow; iRow++) { + int64_t uid = pCommitter->dReader.bDatal.aUid[iRow]; + if (uid == pTbData->uid) { + nRowLeft++; + } + } + } - pRow = tsdbTbDataIterGet(pIter); - ASSERT(pRow == NULL || TSDBROW_TS(pRow) > pCommitter->maxKey); + while (nRowLeft) { + code = tsdbMergeCommitLast(pCommitter, pIter, &nRowLeft); + if (code) goto _err; } // end @@ -888,174 +1048,6 @@ _err: return code; } -#if 0 -static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { - int32_t code = 0; - STbDataIter iter = {0}; - STbDataIter *pIter = &iter; - TSDBROW *pRow; - int32_t iBlock; - int32_t nBlock; - int64_t suid; - int64_t uid; - SBlockIdx *pBlockIdx = NULL; - - if (pTbData) { - tsdbTbDataIterOpen(pTbData, &(TSDBKEY){.ts = pCommitter->minKey, .version = VERSION_MIN}, 0, pIter); - pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) pRow = NULL; - - suid = pTbData->suid; - uid = pTbData->uid; - } else { - pIter = NULL; - pRow = NULL; - } - - if (pBlockIdx) { - code = tsdbReadBlock(pCommitter->dReader.pReader, pBlockIdx, &pCommitter->dReader.mBlock, NULL); - if (code) goto _err; - - nBlock = pCommitter->dReader.mBlock.nItem; - ASSERT(nBlock > 0); - - suid = pBlockIdx->suid; - uid = pBlockIdx->uid; - } else { - nBlock = 0; - } - - if (pRow == NULL && nBlock == 0) goto _exit; - - // start =========== - tMapDataReset(&pCommitter->dWriter.mBlock); - SBlock block; - SBlock *pBlock = █ - - iBlock = 0; - if (iBlock < nBlock) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); - } else { - pBlock = NULL; - } - - if (pRow) { - code = tsdbCommitterUpdateTableSchema(pCommitter, pTbData->suid, pTbData->uid, pTbData->maxSkmVer); - if (code) goto _err; - } - - // merge =========== - while (true) { - if (pRow == NULL && pBlock == NULL) break; - - if (pRow && pBlock) { - if (pBlock->last) { - code = tsdbMergeTableData(pCommitter, pIter, pBlock, - (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = VERSION_MIN}, 0); - if (code) goto _err; - - pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) pRow = NULL; - iBlock++; - if (iBlock < nBlock) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); - } else { - pBlock = NULL; - } - - ASSERT(pRow == NULL && pBlock == NULL); - } else { - int32_t c = tBlockCmprFn(&(SBlock){.maxKey = TSDBROW_KEY(pRow), .minKey = TSDBROW_KEY(pRow)}, pBlock); - if (c > 0) { - // only disk data - code = tsdbCommitTableDiskData(pCommitter, pBlock, pBlockIdx); - if (code) goto _err; - - iBlock++; - if (iBlock < nBlock) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); - } else { - pBlock = NULL; - } - } else if (c < 0) { - // only memory data - code = tsdbCommitTableMemData(pCommitter, pIter, pBlock->minKey, 1); - if (code) goto _err; - - pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) pRow = NULL; - } else { - // merge memory and disk - int32_t nOvlp = tsdbGetOvlpNRow(pIter, pBlock); - ASSERT(nOvlp); - if (pBlock->nRow + nOvlp <= pCommitter->maxRow && pBlock->nSubBlock < TSDB_MAX_SUBBLOCKS) { - code = tsdbMergeAsSubBlock(pCommitter, pIter, pBlock); - if (code) goto _err; - } else { - TSDBKEY toKey = {.ts = pCommitter->maxKey + 1, .version = VERSION_MIN}; - int8_t toDataOnly = 0; - - if (iBlock < nBlock - 1) { - toDataOnly = 1; - - SBlock nextBlock = {0}; - tBlockReset(&nextBlock); - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock + 1, &nextBlock, tGetBlock); - toKey = nextBlock.minKey; - } - - code = tsdbMergeTableData(pCommitter, pIter, pBlock, toKey, toDataOnly); - if (code) goto _err; - } - - pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) pRow = NULL; - iBlock++; - if (iBlock < nBlock) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); - } else { - pBlock = NULL; - } - } - } - } else if (pBlock) { - code = tsdbCommitTableDiskData(pCommitter, pBlock, pBlockIdx); - if (code) goto _err; - - iBlock++; - if (iBlock < nBlock) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); - } else { - pBlock = NULL; - } - } else { - code = - tsdbCommitTableMemData(pCommitter, pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = VERSION_MIN}, 0); - if (code) goto _err; - - pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) pRow = NULL; - ASSERT(pRow == NULL); - } - } - - // end ===================== - code = tsdbCommitTableDataEnd(pCommitter, suid, uid); - if (code) goto _err; - -_exit: - if (pIter) { - pRow = tsdbTbDataIterGet(pIter); - if (pRow) pCommitter->nextKey = TMIN(pCommitter->nextKey, TSDBROW_TS(pRow)); - } - return code; - -_err: - tsdbError("vgId:%d, tsdb commit table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); - return code; -} -#endif - static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { int32_t code = 0; @@ -1122,52 +1114,53 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { } // last - SBlockL blockL; while (true) { - if (pCommitter->dReader.pRow == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRow, &toTable) >= 0) break; + if (pCommitter->dReader.pRowInfo == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRowInfo, &toTable) >= 0) break; - // check if same suid - if (pCommitter->dReader.pRow->suid == 0) { - if (pCommitter->dReader.pRow->uid != 0 /*todo*/) { - // code = tsdbCommitBlockDataL(pCommitter); + // commit if not same schema + if (pCommitter->dWriter.bDatal.nRow > 0) { + if (pCommitter->dWriter.bDatal.suid != pCommitter->dReader.pRowInfo->suid || + pCommitter->dWriter.bDatal.suid == 0) { + code = tsdbCommitLastBlock(pCommitter); if (code) goto _err; } - } else if (pCommitter->dReader.pRow->suid != 0 /*todo*/) { - // code = tsdbCommitBlockDataL(pCommitter); - if (code) goto _err; - } - - // append - code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &pCommitter->dReader.pRow->row, NULL); - if (code) goto _err; - - // next - pCommitter->dReader.iRow++; - if (pCommitter->dReader.iRow < pCommitter->dReader.bDatal.nRow) { - pCommitter->dReader.pRow->uid = pCommitter->dReader.bDatal.aUid[pCommitter->dReader.iRow]; - pCommitter->dReader.pRow->row = tsdbRowFromBlockData(&pCommitter->dReader.bDatal, pCommitter->dReader.iRow); - } else { - pCommitter->dReader.iBlockL++; - if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { - pCommitter->dReader.pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); - - // TODO: code = tsdbReadBlockData(pCommitter->dReader.pReader, NULL, pBlockL, &pCommitter->dReader.bDatal, NULL, - // NULL); - if (code) goto _err; - - pCommitter->dReader.iRow = 0; - pCommitter->dReader.pRow->suid = pCommitter->dReader.pBlockL->suid; - pCommitter->dReader.pRow->uid = pCommitter->dReader.bDatal.aUid[pCommitter->dReader.iRow]; - pCommitter->dReader.pRow->row = tsdbRowFromBlockData(&pCommitter->dReader.bDatal, pCommitter->dReader.iRow); - } else { - pCommitter->dReader.pRow = NULL; - } } - // write - if (pCommitter->dWriter.bDatal.nRow >= pCommitter->maxRow) { - // code = tsdbCommitBlockDataL(pCommitter); + if (pCommitter->dWriter.bDatal.nRow == 0) { + code = tsdbCommitterUpdateTableSchema(pCommitter, pCommitter->dReader.pRowInfo->suid, + pCommitter->dReader.pRowInfo->suid, 1 /*TODO*/); if (code) goto _err; + + pCommitter->dWriter.bDatal.suid = pCommitter->dReader.pRowInfo->suid; + code = tBlockDataSetSchema(&pCommitter->dWriter.bDatal, pCommitter->skmTable.pTSchema); + if (code) goto _err; + } + + // check if it can make sure that one table data in one block + int64_t uid = pCommitter->dReader.pRowInfo->uid; + int32_t nRow = 0; + for (int32_t iRow = pCommitter->dReader.iRow; + (iRow < pCommitter->dReader.bDatal.nRow) && (pCommitter->dReader.bDatal.aUid[iRow] == uid); iRow++) { + nRow++; + } + + ASSERT(nRow > 0 && nRow < pCommitter->minRow); + + if (pCommitter->dWriter.bDatal.nRow + nRow > pCommitter->maxRow) { + ASSERT(pCommitter->dWriter.bDatal.nRow > 0); + + code = tsdbCommitLastBlock(pCommitter); + if (code) goto _err; + } + + while (nRow > 0) { + code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &pCommitter->dReader.pRowInfo->row, NULL); + if (code) goto _err; + + code = tsdbCommitNextLastRow(pCommitter); + if (code) goto _err; + + nRow--; } } @@ -1192,7 +1185,7 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) { STbData *pTbData = (STbData *)taosArrayGetP(pMemTable->aTbData, iTbData); // move commit until current (suid, uid) - code = tsdbMoveCommitData(pCommitter, (TABLEID){.suid = pTbData->suid, .uid = pTbData->uid}); + code = tsdbMoveCommitData(pCommitter, *(TABLEID *)pTbData); if (code) goto _err; // commit current table data @@ -1204,7 +1197,7 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) { if (code) goto _err; if (pCommitter->dWriter.bDatal.nRow > 0) { - // code = tsdbCommitBlockDataL(pCommitter); + code = tsdbCommitLastBlock(pCommitter); if (code) goto _err; } @@ -1261,15 +1254,15 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { goto _exit; } + code = tBlockDataInit(&pCommitter->dReader.bData); + if (code) goto _exit; + pCommitter->dReader.aBlockL = taosArrayInit(0, sizeof(SBlockL)); if (pCommitter->dReader.aBlockL == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - code = tBlockDataInit(&pCommitter->dReader.bData); - if (code) goto _exit; - code = tBlockDataInit(&pCommitter->dReader.bDatal); if (code) goto _exit; @@ -1299,9 +1292,9 @@ _exit: static void tsdbCommitDataEnd(SCommitter *pCommitter) { // Reader taosArrayDestroy(pCommitter->dReader.aBlockIdx); - taosArrayDestroy(pCommitter->dReader.aBlockL); tMapDataClear(&pCommitter->dReader.mBlock); tBlockDataClear(&pCommitter->dReader.bData, 1); + taosArrayDestroy(pCommitter->dReader.aBlockL); tBlockDataClear(&pCommitter->dReader.bDatal, 1); // Writer @@ -1329,9 +1322,6 @@ static int32_t tsdbCommitData(SCommitter *pCommitter) { // impl ==================== pCommitter->nextKey = pMemTable->minKey; while (pCommitter->nextKey < TSKEY_MAX) { - pCommitter->commitFid = tsdbKeyFid(pCommitter->nextKey, pCommitter->minutes, pCommitter->precision); - tsdbFidKeyRange(pCommitter->commitFid, pCommitter->minutes, pCommitter->precision, &pCommitter->minKey, - &pCommitter->maxKey); code = tsdbCommitFileData(pCommitter); if (code) goto _err; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 22615b9508..79929c44a0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -543,9 +543,13 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppB uint32_t delimiter; SBlockIdx blockIdx; - if (!ppBuf) ppBuf = &pBuf; + taosArrayClear(aBlockIdx); + if (size == 0) { + goto _exit; + } // alloc + if (!ppBuf) ppBuf = &pBuf; code = tRealloc(ppBuf, size); if (code) goto _err; @@ -576,7 +580,6 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppB n = tGetU32(*ppBuf + n, &delimiter); ASSERT(delimiter == TSDB_FILE_DLMT); - taosArrayClear(aBlockIdx); while (n < size - sizeof(TSCKSUM)) { n += tGetBlockIdx(*ppBuf + n, &blockIdx); @@ -588,6 +591,7 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppB ASSERT(n + sizeof(TSCKSUM) == size); +_exit: tFree(pBuf); return code; @@ -606,9 +610,13 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf) uint8_t *pBuf = NULL; SBlockL blockl; - if (!ppBuf) ppBuf = &pBuf; + taosArrayClear(aBlockL); + if (size == 0) { + goto _exit; + } // alloc + if (!ppBuf) ppBuf = &pBuf; code = tRealloc(ppBuf, size); if (code) goto _err; @@ -639,7 +647,6 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf) n = tGetU32(*ppBuf + n, &delimiter); ASSERT(delimiter == TSDB_FILE_DLMT); - taosArrayClear(aBlockL); while (n < size - sizeof(TSCKSUM)) { n += tGetBlockL(*ppBuf + n, &blockl); @@ -651,11 +658,13 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf) ASSERT(n + sizeof(TSCKSUM) == size); +_exit: tFree(pBuf); return code; _err: tsdbError("vgId:%d read blockl failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + tFree(pBuf); return code; } @@ -1562,7 +1571,11 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **pp int64_t size; int64_t n; - if (!ppBuf) ppBuf = &pBuf; + // check + if (taosArrayGetSize(aBlockIdx) == 0) { + pHeadFile->offset = pHeadFile->size; + goto _exit; + } // prepare size = tPutU32(NULL, TSDB_FILE_DLMT); @@ -1572,6 +1585,7 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **pp size += sizeof(TSCKSUM); // alloc + if (!ppBuf) ppBuf = &pBuf; code = tRealloc(ppBuf, size); if (code) goto _err; @@ -1596,6 +1610,7 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **pp pHeadFile->offset = pHeadFile->size; pHeadFile->size += size; +_exit: tFree(pBuf); return code; @@ -1662,6 +1677,12 @@ int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL, uint8_t **ppBuf) int64_t size; int64_t n; + // check + if (taosArrayGetSize(aBlockL) == 0) { + pHeadFile->loffset = pHeadFile->size; + goto _exit; + } + // size size = sizeof(uint32_t); for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aBlockL); iBlockL++) { @@ -1695,10 +1716,13 @@ int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL, uint8_t **ppBuf) pHeadFile->loffset = pHeadFile->size; pHeadFile->size += size; +_exit: + tFree(pBuf); return code; _err: tsdbError("vgId:%d tsdb write blockl failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + tFree(pBuf); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 4e0d1b2402..205d8b29dc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1122,6 +1122,7 @@ int32_t tBlockDataInit(SBlockData *pBlockData) { int32_t code = 0; pBlockData->nRow = 0; + pBlockData->aUid = NULL; pBlockData->aVersion = NULL; pBlockData->aTSKEY = NULL; pBlockData->aIdx = taosArrayInit(0, sizeof(int32_t)); @@ -1146,6 +1147,7 @@ void tBlockDataReset(SBlockData *pBlockData) { } void tBlockDataClear(SBlockData *pBlockData, int8_t deepClear) { + tFree((uint8_t *)pBlockData->aUid); tFree((uint8_t *)pBlockData->aVersion); tFree((uint8_t *)pBlockData->aTSKEY); taosArrayDestroy(pBlockData->aIdx); From 7075244183a133bcac4973341f014f5be394225c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 3 Aug 2022 12:16:57 +0000 Subject: [PATCH 05/92] more last file refact --- source/dnode/vnode/src/inc/tsdb.h | 3 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 485 +++++++++--------- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 4 +- 3 files changed, 240 insertions(+), 252 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index af256c0e19..4c31b3e07d 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -469,7 +469,8 @@ struct SColData { }; struct SBlockData { - int64_t suid; + int64_t suid; // 0 means normal table data block + int64_t uid; // 0 means block data in .last file, others in .data file int32_t nRow; int64_t *aUid; int64_t *aVersion; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 70892acb0a..63aa5f3ec8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -290,7 +290,7 @@ _err: return code; } -static int32_t tsdbCommitNextLastRow(SCommitter *pCommitter) { +static int32_t tsdbCommitterNextLastRow(SCommitter *pCommitter) { int32_t code = 0; ASSERT(pCommitter->dReader.pReader); @@ -363,7 +363,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { pCommitter->dReader.bDatal.nRow = 0; pCommitter->dReader.iRow = -1; pCommitter->dReader.pRowInfo = &pCommitter->dReader.rowInfo; - code = tsdbCommitNextLastRow(pCommitter); + code = tsdbCommitterNextLastRow(pCommitter); if (code) goto _err; } else { pCommitter->dReader.pBlockIdx = NULL; @@ -527,128 +527,94 @@ _exit: return code; } -static int32_t tsdbMergeTableData(SCommitter *pCommitter, STbDataIter *pIter, SBlock *pBlockMerge, TSDBKEY toKey, - int8_t toDataOnly) { +static int32_t tsdbMergeCommitData(SCommitter *pCommitter, STbDataIter *pIter, SBlock *pBlock) { int32_t code = 0; - SBlockIdx *pBlockIdx = &(SBlockIdx){.suid = pIter->pTbData->suid, .uid = pIter->pTbData->uid}; - SBlockData *pBlockDataMerge = &pCommitter->dReader.bData; - SBlockData *pBlockData = &pCommitter->dWriter.bData; - SBlock block; - SBlock *pBlock = █ - TSDBROW *pRow1; - TSDBROW row2; - TSDBROW *pRow2 = &row2; + STbData *pTbData = pIter->pTbData; + SBlockData *pBlockDataR = &pCommitter->dReader.bData; + SBlockData *pBlockDataW = &pCommitter->dWriter.bData; - // read SBlockData - code = tsdbReadBlockData(pCommitter->dReader.pReader, pBlockIdx, pBlockMerge, pBlockDataMerge, NULL, NULL); + code = tsdbReadDataBlock(pCommitter->dReader.pReader, pBlock, pBlockDataR, NULL, NULL); if (code) goto _err; - code = tBlockDataSetSchema(pBlockData, pCommitter->skmTable.pTSchema); - if (code) goto _err; + tBlockDataClearData(pBlockDataW); + int32_t iRow = 0; + TSDBROW row; + TSDBROW *pRow1 = tsdbTbDataIterGet(pIter); + TSDBROW *pRow2 = &row; + *pRow2 = tsdbRowFromBlockData(pBlockDataR, iRow); + while (pRow1 && pRow2) { + int32_t c = tsdbRowCmprFn(pRow1, pRow2); - // loop to merge - pRow1 = tsdbTbDataIterGet(pIter); - *pRow2 = tsdbRowFromBlockData(pBlockDataMerge, 0); - ASSERT(pRow1 && tsdbKeyCmprFn(&TSDBROW_KEY(pRow1), &toKey) < 0); - ASSERT(tsdbKeyCmprFn(&TSDBROW_KEY(pRow2), &toKey) < 0); - code = tsdbCommitterUpdateRowSchema(pCommitter, pBlockIdx->suid, pBlockIdx->uid, TSDBROW_SVERSION(pRow1)); - if (code) goto _err; + if (c < 0) { + code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow1)); + if (code) goto _err; - tBlockReset(pBlock); - tBlockDataClearData(pBlockData); - while (true) { - if (pRow1 == NULL && pRow2 == NULL) { - if (pBlockData->nRow == 0) { - break; + code = tBlockDataAppendRow(pBlockDataW, pRow1, pCommitter->skmRow.pTSchema); + if (code) goto _err; + + // next + tsdbTbDataIterNext(pIter); + pRow1 = tsdbTbDataIterGet(pIter); + } else if (c > 0) { + code = tBlockDataAppendRow(pBlockDataW, pRow2, NULL); + if (code) goto _err; + + iRow++; + if (iRow < pBlockDataR->nRow) { + *pRow2 = tsdbRowFromBlockData(pBlockDataR, iRow); } else { - goto _write_block; + pRow2 = NULL; } - } - - if (pRow1 && pRow2) { - int32_t c = tsdbRowCmprFn(pRow1, pRow2); - if (c < 0) { - goto _append_mem_row; - } else if (c > 0) { - goto _append_block_row; - } else { - ASSERT(0); - } - } else if (pRow1) { - goto _append_mem_row; } else { - goto _append_block_row; + ASSERT(0); } - _append_mem_row: - code = tBlockDataAppendRow(pBlockData, pRow1, pCommitter->skmRow.pTSchema); + // check + if (pBlockDataW->nRow >= pCommitter->maxRow * 4 / 5) { + code = tsdbCommitDataBlock(pCommitter); + if (code) goto _err; + } + } + + while (pRow2) { + code = tBlockDataAppendRow(pBlockDataW, pRow2, NULL); if (code) goto _err; - tsdbTbDataIterNext(pIter); - pRow1 = tsdbTbDataIterGet(pIter); - if (pRow1) { - if (tsdbKeyCmprFn(&TSDBROW_KEY(pRow1), &toKey) < 0) { - code = tsdbCommitterUpdateRowSchema(pCommitter, pBlockIdx->suid, pBlockIdx->uid, TSDBROW_SVERSION(pRow1)); - if (code) goto _err; - } else { - pRow1 = NULL; - } - } - - if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { - goto _write_block; - } else { - continue; - } - - _append_block_row: - code = tBlockDataAppendRow(pBlockData, pRow2, NULL); - if (code) goto _err; - - if (pRow2->iRow + 1 < pBlockDataMerge->nRow) { - *pRow2 = tsdbRowFromBlockData(pBlockDataMerge, pRow2->iRow + 1); + iRow++; + if (iRow < pBlockDataR->nRow) { + *pRow2 = tsdbRowFromBlockData(pBlockDataR, iRow); } else { pRow2 = NULL; } - if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { - goto _write_block; - } else { - continue; + // check + if (pBlockDataW->nRow >= pCommitter->maxRow * 4 / 5) { + code = tsdbCommitDataBlock(pCommitter); + if (code) goto _err; } + } - _write_block: - code = tsdbCommitBlockData(pCommitter, pBlockData, pBlock, pBlockIdx, toDataOnly); + // check + if (pBlockDataW->nRow > 0) { + code = tsdbCommitDataBlock(pCommitter); if (code) goto _err; - - tBlockReset(pBlock); - tBlockDataClearData(pBlockData); } return code; _err: - tsdbError("vgId:%d, tsdb merge block and mem failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, tsdb merge commit data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); return code; } -static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter, TSDBKEY toKey, int8_t toDataOnly) { - int32_t code = 0; -#if 0 - TSDBROW *pRow; - SBlock block; - SBlock *pBlock = █ +static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter, TSDBKEY toKey) { + int32_t code = 0; + STbData *pTbData = pIter->pTbData; SBlockData *pBlockData = &pCommitter->dWriter.bData; - int64_t suid = pIter->pTbData->suid; - int64_t uid = pIter->pTbData->uid; - code = tBlockDataSetSchema(pBlockData, pCommitter->skmTable.pTSchema); - if (code) goto _err; - - tBlockReset(pBlock); tBlockDataClearData(pBlockData); - pRow = tsdbTbDataIterGet(pIter); - ASSERT(pRow && tsdbKeyCmprFn(&TSDBROW_KEY(pRow), &toKey) < 0); + + TSDBROW *pRow = tsdbTbDataIterGet(pIter); while (true) { if (pRow == NULL) { if (pBlockData->nRow > 0) { @@ -659,7 +625,7 @@ static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter } // update schema - code = tsdbCommitterUpdateRowSchema(pCommitter, suid, uid, TSDBROW_SVERSION(pRow)); + code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); if (code) goto _err; // append @@ -668,27 +634,20 @@ static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter tsdbTbDataIterNext(pIter); pRow = tsdbTbDataIterGet(pIter); - // if (pRow && tsdbKeyCmprFn(&TSDBROW_KEY(pRow), &toKey) >= 0) pRow = NULL; - // crash on CI, use the block following if (pRow) { - TSDBKEY tmpKey = TSDBROW_KEY(pRow); - if (tsdbKeyCmprFn(&tmpKey, &toKey) >= 0) { + TSDBKEY rowKey = TSDBROW_KEY(pRow); + if (tsdbKeyCmprFn(&rowKey, &toKey) >= 0) { pRow = NULL; } } - if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) goto _write_block; - continue; - - _write_block: - code = tsdbCommitBlockData(pCommitter, pBlockData, pBlock, &(SBlockIdx){.suid = suid, .uid = uid}, toDataOnly); - if (code) goto _err; - - tBlockReset(pBlock); - tBlockDataClearData(pBlockData); + if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { + _write_block: + code = tsdbCommitDataBlock(pCommitter); + if (code) goto _err; + } } -#endif return code; _err: @@ -719,26 +678,6 @@ _err: return code; } -// static int32_t tsdbCommitTableDataEnd(SCommitter *pCommitter, int64_t suid, int64_t uid) { -// int32_t code = 0; -// SBlockIdx blockIdx = {.suid = suid, .uid = uid}; -// SBlockIdx *pBlockIdx = &blockIdx; - -// code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, NULL, pBlockIdx); -// if (code) goto _err; - -// if (taosArrayPush(pCommitter->dWriter.aBlockIdx, pBlockIdx) == NULL) { -// code = TSDB_CODE_OUT_OF_MEMORY; -// goto _err; -// } - -// return code; - -// _err: -// tsdbError("vgId:%d, commit table data end failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); -// return code; -// } - static int32_t tsdbGetNumOfRowsLessThan(STbDataIter *pIter, TSDBKEY key) { int32_t nRow = 0; @@ -763,42 +702,35 @@ static int32_t tsdbGetNumOfRowsLessThan(STbDataIter *pIter, TSDBKEY key) { static int32_t tsdbMergeAsSubBlock(SCommitter *pCommitter, STbDataIter *pIter, SBlock *pBlock) { int32_t code = 0; + STbData *pTbData = pIter->pTbData; SBlockData *pBlockData = &pCommitter->dWriter.bData; - SBlockIdx *pBlockIdx = &(SBlockIdx){.suid = pIter->pTbData->suid, .uid = pIter->pTbData->uid}; - SBlock block; - TSDBROW *pRow; - code = tBlockDataSetSchema(pBlockData, pCommitter->skmTable.pTSchema); - if (code) goto _err; - - pRow = tsdbTbDataIterGet(pIter); - code = tsdbCommitterUpdateRowSchema(pCommitter, pBlockIdx->suid, pBlockIdx->uid, TSDBROW_SVERSION(pRow)); - if (code) goto _err; + tBlockDataClearData(pBlockData); while (true) { - if (pRow == NULL) break; - code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema); - if (code) goto _err; - - tsdbTbDataIterNext(pIter); - pRow = tsdbTbDataIterGet(pIter); + TSDBROW *pRow = tsdbTbDataIterGet(pIter); if (pRow) { - TSDBKEY key = TSDBROW_KEY(pRow); - int32_t c = tBlockCmprFn(&(SBlock){.minKey = key, .maxKey = key}, pBlock); - - if (c == 0) { - code = - tsdbCommitterUpdateRowSchema(pCommitter, pIter->pTbData->suid, pIter->pTbData->uid, TSDBROW_SVERSION(pRow)); - if (code) goto _err; - } else if (c > 0) { + TSDBKEY rowKey = TSDBROW_KEY(pRow); + if (tsdbKeyCmprFn(&rowKey, &pBlock->maxKey) > 0) { pRow = NULL; - } else { - ASSERT(0); } } + + if (pRow == NULL) { + break; + } + + code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); + if (code) goto _err; + + code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema); + if (code) goto _err; } - block = *pBlock; - code = tsdbCommitBlockData(pCommitter, pBlockData, &block, pBlockIdx, 0); + SBlock block = *pBlock; + code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &block, NULL, NULL, pCommitter->cmprAlg); + if (code) goto _err; + + code = tMapDataPutItem(&pCommitter->dWriter.mBlock, &block, tPutBlock); if (code) goto _err; return code; @@ -808,16 +740,101 @@ _err: return code; } -static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter, int32_t *nRow) { - int32_t code = 0; - STbData *pTbData = pIter->pTbData; - TSDBROW *pRow = tsdbTbDataIterGet(pIter); - SRowInfo *pRowInfo = pCommitter->dReader.pRowInfo; +static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { + int32_t code = 0; + STbData *pTbData = pIter->pTbData; + int32_t nRow = tsdbGetNumOfRowsLessThan(pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = VERSION_MIN}); - while (pRow && pRowInfo) { - int32_t c = tsdbRowCmprFn(pRow, &pRowInfo->row); - if (c < 0) { - code = tBlockDataAppendRow(&pCommitter->dWriter.bData, pRow, NULL); + if (pCommitter->dReader.pRowInfo) { + for (int32_t iRow = pCommitter->dReader.iRow; iRow < pCommitter->dReader.bDatal.nRow; iRow++) { + if (pTbData->uid != pCommitter->dReader.bDatal.aUid[iRow]) break; + nRow++; + } + } + + if (nRow == 0) goto _exit; + + SBlockData *pBlockData; + TSDBROW *pRow = tsdbTbDataIterGet(pIter); + SRowInfo *pRowInfo = pCommitter->dReader.pRowInfo; + + if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { + pRow = NULL; + } + + if (pRowInfo && (pRowInfo->suid != pTbData->suid || pRowInfo->uid != pTbData->uid)) { + pRowInfo = NULL; + } + + while (nRow) { + if (nRow < pCommitter->minRow) { // to .last + pBlockData = &pCommitter->dWriter.bDatal; + + // check if same schema + if (pBlockData->suid != pTbData->suid || pBlockData->suid == 0) { + code = tsdbCommitLastBlock(pCommitter); + if (code) goto _err; + + code = tBlockDataSetSchema(pBlockData, pCommitter->skmTable.pTSchema); + if (code) goto _err; + } + + if (pBlockData->nRow + nRow > pCommitter->maxRow) { + code = tsdbCommitLastBlock(pCommitter); + if (code) goto _err; + } + } else { // to .data + pBlockData = &pCommitter->dWriter.bData; + } + + while (pRow && pRowInfo) { + int32_t c = tsdbRowCmprFn(pRow, &pRowInfo->row); + if (c < 0) { + code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); + if (code) goto _err; + + code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema); + if (code) goto _err; + + tsdbTbDataIterNext(pIter); + pRow = tsdbTbDataIterGet(pIter); + if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { + pRow = NULL; + } + } else if (c > 0) { + code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL); + if (code) goto _err; + + code = tsdbCommitterNextLastRow(pCommitter); + if (code) goto _err; + + pRowInfo = pCommitter->dReader.pRowInfo; + if (pRowInfo && (pRowInfo->suid != pTbData->suid || pRowInfo->uid != pTbData->uid)) { + pRowInfo = NULL; + } + } else { + ASSERT(0); + } + + nRow--; + + if (pBlockData->uid) { // .data block + if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { + code = tsdbCommitDataBlock(pCommitter); + if (code) goto _err; + + goto _outer_break; + } + } else { + ASSERT(pBlockData->nRow <= pCommitter->maxRow); + } + } + + while (pRow) { + code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); + if (code) goto _err; + + code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema); if (code) goto _err; tsdbTbDataIterNext(pIter); @@ -825,81 +842,54 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter, i if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { pRow = NULL; } - } else if (c > 0) { - code = tBlockDataAppendRow(&pCommitter->dWriter.bData, &pRowInfo->row, NULL); + + nRow--; + if (pBlockData->uid) { // .data block + if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { + code = tsdbCommitDataBlock(pCommitter); + if (code) goto _err; + + goto _outer_break; + } + } else { + ASSERT(pBlockData->nRow <= pCommitter->maxRow); + } + } + + while (pRowInfo) { + code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL); if (code) goto _err; - pCommitter->dReader.iRow++; - if (pCommitter->dReader.iRow < pCommitter->dReader.bDatal.nRow) { - // todo - } else { - pCommitter->dReader.iBlockL++; - if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { - // todo - } else { - // todo - } - } + code = tsdbCommitterNextLastRow(pCommitter); + if (code) goto _err; pRowInfo = pCommitter->dReader.pRowInfo; if (pRowInfo && (pRowInfo->suid != pTbData->suid || pRowInfo->uid != pTbData->uid)) { pRowInfo = NULL; } - } else { - ASSERT(0); - } - (*nRow)--; - ASSERT(*nRow >= 0); - if (pCommitter->dWriter.bData.nRow >= pCommitter->maxRow * 4 / 5) goto _write_block_data; - } + nRow--; + if (pBlockData->uid) { // .data block + if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { + code = tsdbCommitDataBlock(pCommitter); + if (code) goto _err; - while (pRow) { - code = tBlockDataAppendRow(&pCommitter->dWriter.bData, pRow, NULL); - if (code) goto _err; - - tsdbTbDataIterNext(pIter); - pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { - pRow = NULL; - } - - (*nRow)--; - ASSERT(*nRow >= 0); - if (pCommitter->dWriter.bData.nRow >= pCommitter->maxRow * 4 / 5) goto _write_block_data; - } - - while (pRowInfo) { - code = tBlockDataAppendRow(&pCommitter->dWriter.bData, &pRowInfo->row, NULL); - if (code) goto _err; - - pCommitter->dReader.iRow++; - if (pCommitter->dReader.iRow < pCommitter->dReader.bDatal.nRow) { - // todo - } else { - pCommitter->dReader.iBlockL++; - if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { - // todo + goto _outer_break; + } } else { - // todo + ASSERT(pBlockData->nRow <= pCommitter->maxRow); } } - pRowInfo = pCommitter->dReader.pRowInfo; - if (pRowInfo && (pRowInfo->suid != pTbData->suid || pRowInfo->uid != pTbData->uid)) { - pRowInfo = NULL; - } - - (*nRow)--; - ASSERT(*nRow >= 0); - if (pCommitter->dWriter.bData.nRow >= pCommitter->maxRow * 4 / 5) goto _write_block_data; + _outer_break: + ASSERT(nRow >= 0); } - SBlock block; -_write_block_data: +_exit: return code; _err: + tsdbError("vgId:%d tsdb merge commit last failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); return code; } @@ -909,14 +899,6 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { ASSERT(pCommitter->dReader.pBlockIdx == NULL || tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, pTbData) >= 0); ASSERT(pCommitter->dReader.pRowInfo == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRowInfo, pTbData) >= 0); - // end last if need - if (pTbData->suid == 0 || pTbData->suid != 0 /*todo*/) { - if (pCommitter->dWriter.bDatal.nRow > 0) { - // TODO: code = tsdbCommitBlockDataL(pCommitter); - if (code) goto _err; - } - } - // merge commit table data STbDataIter iter = {0}; STbDataIter *pIter = &iter; @@ -930,22 +912,22 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { if (pRow == NULL) goto _exit; - SBlockIdx *pBlockIdx = NULL; - int32_t iBlock = 0; - SBlock block; - SBlock *pBlock = █ - - if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pBlockIdx) == 0) { - pBlockIdx = pCommitter->dReader.pBlockIdx; - } - - if (pBlockIdx && iBlock < pCommitter->dReader.mBlock.nItem) { + int32_t iBlock = 0; + SBlock block; + SBlock *pBlock = █ + if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pBlockIdx) == 0 && + iBlock < pCommitter->dReader.mBlock.nItem) { tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); } else { pBlock = NULL; } + code = tsdbCommitterUpdateTableSchema(pCommitter, pTbData->suid, pTbData->uid, pTbData->maxSkmVer); + if (code) goto _err; + tMapDataReset(&pCommitter->dWriter.mBlock); + code = tBlockDataSetSchema(&pCommitter->dWriter.bData, pCommitter->skmTable.pTSchema); + if (code) goto _err; while (pBlock && pRow) { int32_t c = tBlockCmprFn(pBlock, &(SBlock){.minKey = TSDBROW_KEY(pRow), .maxKey = TSDBROW_KEY(pRow)}); if (c < 0) { // disk @@ -959,8 +941,8 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { } else { pBlock = NULL; } - } else if (c < 0) { // memory - code = tsdbCommitTableMemData(pCommitter, pIter, pBlock->minKey, 1); + } else if (c > 0) { // memory + code = tsdbCommitTableMemData(pCommitter, pIter, pBlock->minKey); if (code) goto _err; // next @@ -977,7 +959,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { code = tsdbMergeAsSubBlock(pCommitter, pIter, pBlock); if (code) goto _err; } else { - // code = tsdbMergeTableData(pCommitter, pIter, pBlock, NULL, 1); + code = tsdbMergeCommitData(pCommitter, pIter, pBlock); if (code) goto _err; } @@ -999,6 +981,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock); if (code) goto _err; + // next iBlock++; if (iBlock < pCommitter->dReader.mBlock.nItem) { tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); @@ -1008,8 +991,11 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { } // merge with last - int32_t nRowLeft = tsdbGetNumOfRowsLessThan(pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = VERSION_MIN}); - if (pCommitter->dReader.pRowInfo) { + code = tsdbMergeCommitLast(pCommitter, pIter); + if (code) goto _err; +#if 0 + int32_t nRowLeft = tsdbGetNumOfRowsLessThan(pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = + VERSION_MIN}); if (pCommitter->dReader.pRowInfo) { for (int32_t iRow = pCommitter->dReader.iRow; iRow < pCommitter->dReader.bDatal.nRow; iRow++) { int64_t uid = pCommitter->dReader.bDatal.aUid[iRow]; if (uid == pTbData->uid) { @@ -1022,6 +1008,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { code = tsdbMergeCommitLast(pCommitter, pIter, &nRowLeft); if (code) goto _err; } +#endif // end if (pCommitter->dWriter.mBlock.nItem > 0) { @@ -1051,14 +1038,14 @@ _err: static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { int32_t code = 0; - // write aBlockL - code = tsdbWriteBlockL(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockL, NULL); - if (code) goto _err; - // write aBlockIdx code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx, NULL); if (code) goto _err; + // write aBlockL + code = tsdbWriteBlockL(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockL, NULL); + if (code) goto _err; + // update file header code = tsdbUpdateDFileSetHeader(pCommitter->dWriter.pWriter); if (code) goto _err; @@ -1157,7 +1144,7 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &pCommitter->dReader.pRowInfo->row, NULL); if (code) goto _err; - code = tsdbCommitNextLastRow(pCommitter); + code = tsdbCommitterNextLastRow(pCommitter); if (code) goto _err; nRow--; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 79929c44a0..9634b14bf7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -537,7 +537,7 @@ _err: int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppBuf) { int32_t code = 0; int64_t offset = pReader->pSet->pHeadF->offset; - int64_t size = pReader->pSet->pHeadF->size - offset; + int64_t size = pReader->pSet->pHeadF->loffset - offset; uint8_t *pBuf = NULL; int64_t n; uint32_t delimiter; @@ -604,7 +604,7 @@ _err: int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf) { int32_t code = 0; int64_t offset = pReader->pSet->pHeadF->loffset; - int64_t size = pReader->pSet->pHeadF->offset - offset; + int64_t size = pReader->pSet->pHeadF->size - offset; int64_t n; uint32_t delimiter; uint8_t *pBuf = NULL; From e6d374b0066eb0011fab2f298453d71ee6caaa69 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 3 Aug 2022 12:19:13 +0000 Subject: [PATCH 06/92] make it compile --- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 9634b14bf7..8417b0d1b8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -729,6 +729,21 @@ _err: return code; } +int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, uint8_t **ppBuf1, + uint8_t **ppBuf2) { + int32_t code = 0; + // TODO + return code; +} + +int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, uint8_t **ppBuf1, + uint8_t **ppBuf2) { + int32_t code = 0; + ASSERT(0); + // TODO + return code; +} + static int32_t tsdbReadBlockDataKey(SBlockData *pBlockData, SSubBlock *pSubBlock, uint8_t *pBuf, uint8_t **ppBuf) { int32_t code = 0; int64_t size = pSubBlock->szVersion + pSubBlock->szTSKEY + sizeof(TSCKSUM); @@ -2098,6 +2113,20 @@ _err: return code; } +int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock *pBlock, uint8_t **ppBuf1, + uint8_t **ppBuf2, int8_t cmprAlg) { + int32_t code = 0; + // TODO + return code; +} + +int32_t tsdbWriteLastBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockL *pBlockL, uint8_t **ppBuf1, + uint8_t **ppBuf2, int8_t cmprAlg) { + int32_t code = 0; + // TODO + return code; +} + int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { int32_t code = 0; int64_t n; From d1c7e3a531cffe060c49eee955d9da4e75c6a456 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 4 Aug 2022 05:42:15 +0000 Subject: [PATCH 07/92] more last refact --- source/dnode/vnode/src/inc/tsdb.h | 29 +++-- source/dnode/vnode/src/tsdb/tsdbCommit.c | 25 +++-- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 4 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 104 +++++++++--------- 4 files changed, 90 insertions(+), 72 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 4c31b3e07d..29626ed67d 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -134,14 +134,15 @@ int32_t tGetColData(uint8_t *p, SColData *pColData); #define tBlockDataLastRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1) #define tBlockDataFirstKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataFirstRow(PBLOCKDATA)) #define tBlockDataLastKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA)) -int32_t tBlockDataInit(SBlockData *pBlockData); -void tBlockDataReset(SBlockData *pBlockData); -int32_t tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema); +int32_t tBlockDataInit(SBlockData *pBlockData); +void tBlockDataClear(SBlockData *pBlockData, int8_t deepClear); +void tBlockDataReset(SBlockData *pBlockData); +int32_t tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema, int64_t suid, int64_t uid); +int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); +int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); + int32_t tBlockDataCorrectSchema(SBlockData *pBlockData, SBlockData *pBlockDataFrom); void tBlockDataClearData(SBlockData *pBlockData); -void tBlockDataClear(SBlockData *pBlockData, int8_t deepClear); -int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); -int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema); int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData); int32_t tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest); SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx); @@ -468,13 +469,17 @@ struct SColData { uint8_t *pData; }; +// (SBlockData){.suid = 0, .uid = 0}: block data not initialized +// (SBlockData){.suid = suid, .uid = uid}: block data for ONE child table int .data file +// (SBlockData){.suid = suid, .uid = 0}: block data for N child tables int .last file +// (SBlockData){.suid = 0, .uid = uid}: block data for 1 normal table int .last/.data file struct SBlockData { - int64_t suid; // 0 means normal table data block - int64_t uid; // 0 means block data in .last file, others in .data file - int32_t nRow; - int64_t *aUid; - int64_t *aVersion; - TSKEY *aTSKEY; + int64_t suid; // 0 means normal table block data, otherwise child table block data + int64_t uid; // 0 means block data in .last file, otherwise in .data file + int32_t nRow; // number of rows + int64_t *aUid; // uids of each row, only exist in block data in .last file (uid == 0) + int64_t *aVersion; // versions of each row + TSKEY *aTSKEY; // timestamp of each row SArray *aIdx; // SArray SArray *aColData; // SArray }; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 63aa5f3ec8..3f38b664c2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -296,22 +296,28 @@ static int32_t tsdbCommitterNextLastRow(SCommitter *pCommitter) { ASSERT(pCommitter->dReader.pReader); ASSERT(pCommitter->dReader.pRowInfo); + SBlockData *pBlockDatal = &pCommitter->dReader.bDatal; pCommitter->dReader.iRow++; - if (pCommitter->dReader.iRow < pCommitter->dReader.bDatal.nRow) { - pCommitter->dReader.pRowInfo->uid = pCommitter->dReader.bData.aUid[pCommitter->dReader.iRow]; - pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(&pCommitter->dReader.bDatal, pCommitter->dReader.iRow); + if (pCommitter->dReader.iRow < pBlockDatal->nRow) { + if (pBlockDatal->uid == 0) { + pCommitter->dReader.pRowInfo->uid = pBlockDatal->aUid[pCommitter->dReader.iRow]; + } + pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(pBlockDatal, pCommitter->dReader.iRow); } else { pCommitter->dReader.iBlockL++; if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { pCommitter->dReader.pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); - code = tsdbReadLastBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockL, &pCommitter->dReader.bDatal, - NULL, NULL); + code = tsdbReadLastBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockL, pBlockDatal, NULL, NULL); if (code) goto _exit; pCommitter->dReader.iRow = 0; - pCommitter->dReader.pRowInfo->suid = pCommitter->dReader.pBlockL->suid; - pCommitter->dReader.pRowInfo->uid = pCommitter->dReader.bData.aUid[pCommitter->dReader.iRow]; - pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(&pCommitter->dReader.bDatal, pCommitter->dReader.iRow); + pCommitter->dReader.pRowInfo->suid = pBlockDatal->suid; + if (pBlockDatal->uid) { + pCommitter->dReader.pRowInfo->uid = pBlockDatal->uid; + } else { + pCommitter->dReader.pRowInfo->uid = pBlockDatal->aUid[0]; + } + pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(pBlockDatal, pCommitter->dReader.iRow); } else { pCommitter->dReader.pRowInfo = NULL; } @@ -354,15 +360,16 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { } else { pCommitter->dReader.pBlockIdx = NULL; } + tBlockDataReset(&pCommitter->dReader.bData); // last code = tsdbReadBlockL(pCommitter->dReader.pReader, pCommitter->dReader.aBlockL, NULL); if (code) goto _err; pCommitter->dReader.iBlockL = -1; - pCommitter->dReader.bDatal.nRow = 0; pCommitter->dReader.iRow = -1; pCommitter->dReader.pRowInfo = &pCommitter->dReader.rowInfo; + tBlockDataReset(&pCommitter->dReader.bDatal); code = tsdbCommitterNextLastRow(pCommitter); if (code) goto _err; } else { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 8417b0d1b8..73bac77e6d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -591,8 +591,8 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppB ASSERT(n + sizeof(TSCKSUM) == size); -_exit: tFree(pBuf); +_exit: return code; _err: @@ -658,8 +658,8 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf) ASSERT(n + sizeof(TSCKSUM) == size); -_exit: tFree(pBuf); +_exit: return code; _err: diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 205d8b29dc..7beac23912 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1121,6 +1121,8 @@ static FORCE_INLINE int32_t tColDataCmprFn(const void *p1, const void *p2) { int32_t tBlockDataInit(SBlockData *pBlockData) { int32_t code = 0; + pBlockData->suid = 0; + pBlockData->uid = 0; pBlockData->nRow = 0; pBlockData->aUid = NULL; pBlockData->aVersion = NULL; @@ -1141,36 +1143,45 @@ _exit: return code; } -void tBlockDataReset(SBlockData *pBlockData) { - pBlockData->nRow = 0; - taosArrayClear(pBlockData->aIdx); -} - void tBlockDataClear(SBlockData *pBlockData, int8_t deepClear) { tFree((uint8_t *)pBlockData->aUid); tFree((uint8_t *)pBlockData->aVersion); tFree((uint8_t *)pBlockData->aTSKEY); taosArrayDestroy(pBlockData->aIdx); taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataClear : NULL); - pBlockData->aColData = NULL; - pBlockData->aIdx = NULL; - pBlockData->aTSKEY = NULL; + pBlockData->aUid = NULL; pBlockData->aVersion = NULL; + pBlockData->aTSKEY = NULL; + pBlockData->aIdx = NULL; + pBlockData->aColData = NULL; } -int32_t tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema) { - int32_t code = 0; - SColData *pColData; - STColumn *pTColumn; +void tBlockDataReset(SBlockData *pBlockData) { + pBlockData->suid = 0; + pBlockData->uid = 0; + pBlockData->nRow = 0; + taosArrayClear(pBlockData->aIdx); +} + +int32_t tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema, int64_t suid, int64_t uid) { + int32_t code = 0; + + ASSERT(suid || uid); tBlockDataReset(pBlockData); - for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { - pTColumn = &pTSchema->columns[iColumn]; + pBlockData->suid = suid; + pBlockData->uid = uid; - code = tBlockDataAddColData(pBlockData, iColumn - 1, &pColData); - if (code) goto _exit; + if (pTSchema) { + for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { + STColumn *pTColumn = &pTSchema->columns[iColumn]; + SColData *pColData; - tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) != 0); + code = tBlockDataAddColData(pBlockData, iColumn - 1, &pColData); + if (code) goto _exit; + + tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) != 0); + } } _exit: @@ -1211,52 +1222,47 @@ _err: return code; } -int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid) { int32_t code = 0; - // TSDBKEY + ASSERT(pBlockData->suid || pBlockData->uid); + + // uid + if (pBlockData->uid == 0) { + ASSERT(uid); + code = tRealloc((uint8_t **)&pBlockData->aUid, sizeof(int64_t) * (pBlockData->nRow + 1)); + if (code) goto _err; + pBlockData->aUid[pBlockData->nRow] = uid; + } + // version code = tRealloc((uint8_t **)&pBlockData->aVersion, sizeof(int64_t) * (pBlockData->nRow + 1)); if (code) goto _err; + pBlockData->aVersion[pBlockData->nRow] = TSDBROW_VERSION(pRow); + // timestamp code = tRealloc((uint8_t **)&pBlockData->aTSKEY, sizeof(TSKEY) * (pBlockData->nRow + 1)); if (code) goto _err; - pBlockData->aVersion[pBlockData->nRow] = TSDBROW_VERSION(pRow); pBlockData->aTSKEY[pBlockData->nRow] = TSDBROW_TS(pRow); // OTHER - int32_t iColData = 0; - int32_t nColData = taosArrayGetSize(pBlockData->aIdx); - SRowIter iter = {0}; - SRowIter *pIter = &iter; - SColData *pColData; - SColVal *pColVal; + SRowIter rIter = {0}; + SColVal *pColVal; - if (nColData == 0) goto _exit; + tRowIterInit(&rIter, pRow, pTSchema); + pColVal = tRowIterNext(&rIter); + for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { + SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - tRowIterInit(pIter, pRow, pTSchema); - pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - pColVal = tRowIterNext(pIter); + while (pColVal && pColVal->cid < pColData->cid) { + pColVal = tRowIterNext(&rIter); + } - while (pColData) { - if (pColVal) { - if (pColData->cid == pColVal->cid) { - code = tColDataAppendValue(pColData, pColVal); - if (code) goto _err; - - pColVal = tRowIterNext(pIter); - pColData = ((++iColData) < nColData) ? tBlockDataGetColDataByIdx(pBlockData, iColData) : NULL; - } else if (pColData->cid < pColVal->cid) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _err; - - pColData = ((++iColData) < nColData) ? tBlockDataGetColDataByIdx(pBlockData, iColData) : NULL; - } else { - pColVal = tRowIterNext(pIter); - } - } else { + if (pColVal == NULL || pColVal->cid > pColData->cid) { code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); if (code) goto _err; - - pColData = ((++iColData) < nColData) ? tBlockDataGetColDataByIdx(pBlockData, iColData) : NULL; + } else { + code = tColDataAppendValue(pColData, pColVal); + if (code) goto _err; + pColVal = tRowIterNext(&rIter); } } From 67db73b56003764532c53bb5553cd81474252675 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 4 Aug 2022 07:58:53 +0000 Subject: [PATCH 08/92] more last refact work --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 296 +++++++++++------------ 2 files changed, 136 insertions(+), 162 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 29626ed67d..4c7dbd0a45 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -140,9 +140,9 @@ void tBlockDataReset(SBlockData *pBlockData); int32_t tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema, int64_t suid, int64_t uid); int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); +void tBlockDataClearData(SBlockData *pBlockData); int32_t tBlockDataCorrectSchema(SBlockData *pBlockData, SBlockData *pBlockDataFrom); -void tBlockDataClearData(SBlockData *pBlockData); int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData); int32_t tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest); SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 3f38b664c2..e5c6ee44a0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -299,7 +299,9 @@ static int32_t tsdbCommitterNextLastRow(SCommitter *pCommitter) { SBlockData *pBlockDatal = &pCommitter->dReader.bDatal; pCommitter->dReader.iRow++; if (pCommitter->dReader.iRow < pBlockDatal->nRow) { - if (pBlockDatal->uid == 0) { + if (pBlockDatal->uid) { + pCommitter->dReader.pRowInfo->uid = pBlockDatal->uid; + } else { pCommitter->dReader.pRowInfo->uid = pBlockDatal->aUid[pCommitter->dReader.iRow]; } pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(pBlockDatal, pCommitter->dReader.iRow); @@ -327,6 +329,28 @@ _exit: return code; } +static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) { + int32_t code = 0; + + ASSERT(pCommitter->dReader.pBlockIdx); + + pCommitter->dReader.iBlockIdx++; + if (pCommitter->dReader.iBlockIdx < taosArrayGetSize(pCommitter->dReader.aBlockIdx)) { + pCommitter->dReader.pBlockIdx = + (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx); + + code = tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock, NULL); + if (code) goto _exit; + + ASSERT(pCommitter->dReader.mBlock.nItem > 0); + } else { + pCommitter->dReader.pBlockIdx = NULL; + } + +_exit: + return code; +} + static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { int32_t code = 0; STsdb *pTsdb = pCommitter->pTsdb; @@ -469,37 +493,18 @@ _exit: return code; } -static int32_t tsdbCommitBlockData(SCommitter *pCommitter, SBlockData *pBlockData, SBlock *pBlock, SBlockIdx *pBlockIdx, - int8_t toDataOnly) { - int32_t code = 0; - - if (pBlock->nSubBlock == 0) { - if (!toDataOnly && pBlockData->nRow < pCommitter->minRow) { - pBlock->last = 1; - } else { - pBlock->last = 0; - } - } - - code = - tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, NULL, NULL, pBlockIdx, pBlock, pCommitter->cmprAlg); - if (code) goto _err; - - code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock); - if (code) goto _err; - - return code; - -_err: - return code; -} - -static int32_t tsdbCommitDataBlock(SCommitter *pCommitter) { +static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) { int32_t code = 0; SBlock block; ASSERT(pCommitter->dWriter.bData.nRow > 0); + if (pBlock) { + block = *pBlock; // as a subblock + } else { + tBlockReset(&block); // as a new block + } + code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bData, &block, NULL, NULL, pCommitter->cmprAlg); if (code) goto _exit; @@ -556,14 +561,14 @@ static int32_t tsdbMergeCommitData(SCommitter *pCommitter, STbDataIter *pIter, S code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow1)); if (code) goto _err; - code = tBlockDataAppendRow(pBlockDataW, pRow1, pCommitter->skmRow.pTSchema); + code = tBlockDataAppendRow(pBlockDataW, pRow1, pCommitter->skmRow.pTSchema, pTbData->uid); if (code) goto _err; // next tsdbTbDataIterNext(pIter); pRow1 = tsdbTbDataIterGet(pIter); } else if (c > 0) { - code = tBlockDataAppendRow(pBlockDataW, pRow2, NULL); + code = tBlockDataAppendRow(pBlockDataW, pRow2, NULL, pTbData->uid); if (code) goto _err; iRow++; @@ -578,13 +583,13 @@ static int32_t tsdbMergeCommitData(SCommitter *pCommitter, STbDataIter *pIter, S // check if (pBlockDataW->nRow >= pCommitter->maxRow * 4 / 5) { - code = tsdbCommitDataBlock(pCommitter); + code = tsdbCommitDataBlock(pCommitter, NULL); if (code) goto _err; } } while (pRow2) { - code = tBlockDataAppendRow(pBlockDataW, pRow2, NULL); + code = tBlockDataAppendRow(pBlockDataW, pRow2, NULL, pTbData->uid); if (code) goto _err; iRow++; @@ -596,14 +601,14 @@ static int32_t tsdbMergeCommitData(SCommitter *pCommitter, STbDataIter *pIter, S // check if (pBlockDataW->nRow >= pCommitter->maxRow * 4 / 5) { - code = tsdbCommitDataBlock(pCommitter); + code = tsdbCommitDataBlock(pCommitter, NULL); if (code) goto _err; } } // check if (pBlockDataW->nRow > 0) { - code = tsdbCommitDataBlock(pCommitter); + code = tsdbCommitDataBlock(pCommitter, NULL); if (code) goto _err; } @@ -620,7 +625,6 @@ static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter SBlockData *pBlockData = &pCommitter->dWriter.bData; tBlockDataClearData(pBlockData); - TSDBROW *pRow = tsdbTbDataIterGet(pIter); while (true) { if (pRow == NULL) { @@ -636,7 +640,7 @@ static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter if (code) goto _err; // append - code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema); + code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid); if (code) goto _err; tsdbTbDataIterNext(pIter); @@ -650,7 +654,7 @@ static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { _write_block: - code = tsdbCommitDataBlock(pCommitter); + code = tsdbCommitDataBlock(pCommitter, NULL); if (code) goto _err; } } @@ -662,29 +666,6 @@ _err: return code; } -static int32_t tsdbCommitTableDiskData(SCommitter *pCommitter, SBlock *pBlock, SBlockIdx *pBlockIdx) { - int32_t code = 0; - SBlock block; - - if (pBlock->last) { - code = tsdbReadBlockData(pCommitter->dReader.pReader, pBlockIdx, pBlock, &pCommitter->dReader.bData, NULL, NULL); - if (code) goto _err; - - tBlockReset(&block); - code = tsdbCommitBlockData(pCommitter, &pCommitter->dReader.bData, &block, pBlockIdx, 0); - if (code) goto _err; - } else { - code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock); - if (code) goto _err; - } - - return code; - -_err: - tsdbError("vgId:%d, tsdb commit table disk data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); - return code; -} - static int32_t tsdbGetNumOfRowsLessThan(STbDataIter *pIter, TSDBKEY key) { int32_t nRow = 0; @@ -713,31 +694,29 @@ static int32_t tsdbMergeAsSubBlock(SCommitter *pCommitter, STbDataIter *pIter, S SBlockData *pBlockData = &pCommitter->dWriter.bData; tBlockDataClearData(pBlockData); + TSDBROW *pRow = tsdbTbDataIterGet(pIter); while (true) { - TSDBROW *pRow = tsdbTbDataIterGet(pIter); + if (pRow == NULL) break; + + code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); + if (code) goto _err; + + code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid); + if (code) goto _err; + + tsdbTbDataIterNext(pIter); + pRow = tsdbTbDataIterGet(pIter); if (pRow) { TSDBKEY rowKey = TSDBROW_KEY(pRow); if (tsdbKeyCmprFn(&rowKey, &pBlock->maxKey) > 0) { pRow = NULL; } } - - if (pRow == NULL) { - break; - } - - code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); - if (code) goto _err; - - code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema); - if (code) goto _err; } - SBlock block = *pBlock; - code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &block, NULL, NULL, pCommitter->cmprAlg); - if (code) goto _err; + ASSERT(pBlockData->nRow > 0 && pBlock->nRow + pBlockData->nRow <= pCommitter->maxRow); - code = tMapDataPutItem(&pCommitter->dWriter.mBlock, &block, tPutBlock); + code = tsdbCommitDataBlock(pCommitter, pBlock); if (code) goto _err; return code; @@ -753,36 +732,51 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { int32_t nRow = tsdbGetNumOfRowsLessThan(pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = VERSION_MIN}); if (pCommitter->dReader.pRowInfo) { - for (int32_t iRow = pCommitter->dReader.iRow; iRow < pCommitter->dReader.bDatal.nRow; iRow++) { - if (pTbData->uid != pCommitter->dReader.bDatal.aUid[iRow]) break; - nRow++; + if (pCommitter->dReader.pRowInfo->suid) { + for (int32_t iRow = pCommitter->dReader.iRow; iRow < pCommitter->dReader.bDatal.nRow; iRow++) { + if (pTbData->uid != pCommitter->dReader.bDatal.aUid[iRow]) break; + nRow++; + } + } else { + ASSERT(pCommitter->dReader.iRow == 0); + nRow += pCommitter->dReader.bDatal.nRow; } } if (nRow == 0) goto _exit; - SBlockData *pBlockData; - TSDBROW *pRow = tsdbTbDataIterGet(pIter); - SRowInfo *pRowInfo = pCommitter->dReader.pRowInfo; - + TSDBROW *pRow = tsdbTbDataIterGet(pIter); if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { pRow = NULL; } - if (pRowInfo && (pRowInfo->suid != pTbData->suid || pRowInfo->uid != pTbData->uid)) { + SRowInfo *pRowInfo = pCommitter->dReader.pRowInfo; + if (pRowInfo && pRowInfo->uid != pTbData->uid) { pRowInfo = NULL; } while (nRow) { + SBlockData *pBlockData; + int8_t toData; + if (nRow < pCommitter->minRow) { // to .last + toData = 0; pBlockData = &pCommitter->dWriter.bDatal; - // check if same schema - if (pBlockData->suid != pTbData->suid || pBlockData->suid == 0) { - code = tsdbCommitLastBlock(pCommitter); - if (code) goto _err; + // commit and reset block data schema if need + if (pBlockData->nRow > 0) { + if (pBlockData->suid != pTbData->suid || pBlockData->suid == 0) { + code = tsdbCommitLastBlock(pCommitter); + if (code) goto _err; - code = tBlockDataSetSchema(pBlockData, pCommitter->skmTable.pTSchema); + tBlockDataReset(pBlockData); + } + } + + // set block data schema if need + if (pBlockData->suid == 0 && pBlockData->uid == 0) { + code = tBlockDataSetSchema(pBlockData, pCommitter->skmTable.pTSchema, pTbData->suid, + pTbData->suid ? 0 : pTbData->uid); if (code) goto _err; } @@ -791,7 +785,9 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { if (code) goto _err; } } else { // to .data + toData = 1; pBlockData = &pCommitter->dWriter.bData; + ASSERT(pBlockData->nRow == 0); } while (pRow && pRowInfo) { @@ -800,7 +796,7 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); if (code) goto _err; - code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema); + code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid); if (code) goto _err; tsdbTbDataIterNext(pIter); @@ -809,14 +805,14 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { pRow = NULL; } } else if (c > 0) { - code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL); + code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL, pTbData->uid); if (code) goto _err; code = tsdbCommitterNextLastRow(pCommitter); if (code) goto _err; pRowInfo = pCommitter->dReader.pRowInfo; - if (pRowInfo && (pRowInfo->suid != pTbData->suid || pRowInfo->uid != pTbData->uid)) { + if (pRowInfo && pRowInfo->uid != pTbData->uid) { pRowInfo = NULL; } } else { @@ -824,16 +820,12 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { } nRow--; - - if (pBlockData->uid) { // .data block - if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { - code = tsdbCommitDataBlock(pCommitter); + if (toData) { + if (nRow == 0 || pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { + code = tsdbCommitDataBlock(pCommitter, NULL); if (code) goto _err; - goto _outer_break; } - } else { - ASSERT(pBlockData->nRow <= pCommitter->maxRow); } } @@ -841,7 +833,7 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); if (code) goto _err; - code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema); + code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid); if (code) goto _err; tsdbTbDataIterNext(pIter); @@ -851,40 +843,34 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { } nRow--; - if (pBlockData->uid) { // .data block - if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { - code = tsdbCommitDataBlock(pCommitter); + if (toData) { + if (nRow == 0 || pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { + code = tsdbCommitDataBlock(pCommitter, NULL); if (code) goto _err; - goto _outer_break; } - } else { - ASSERT(pBlockData->nRow <= pCommitter->maxRow); } } while (pRowInfo) { - code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL); + code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL, pTbData->uid); if (code) goto _err; code = tsdbCommitterNextLastRow(pCommitter); if (code) goto _err; pRowInfo = pCommitter->dReader.pRowInfo; - if (pRowInfo && (pRowInfo->suid != pTbData->suid || pRowInfo->uid != pTbData->uid)) { + if (pRowInfo && pRowInfo->uid != pTbData->uid) { pRowInfo = NULL; } nRow--; - if (pBlockData->uid) { // .data block - if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { - code = tsdbCommitDataBlock(pCommitter); + if (toData) { + if (nRow == 0 || pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { + code = tsdbCommitDataBlock(pCommitter, NULL); if (code) goto _err; - goto _outer_break; } - } else { - ASSERT(pBlockData->nRow <= pCommitter->maxRow); } } @@ -922,8 +908,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { int32_t iBlock = 0; SBlock block; SBlock *pBlock = █ - if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pBlockIdx) == 0 && - iBlock < pCommitter->dReader.mBlock.nItem) { + if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pBlockIdx) == 0) { tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); } else { pBlock = NULL; @@ -933,8 +918,10 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { if (code) goto _err; tMapDataReset(&pCommitter->dWriter.mBlock); - code = tBlockDataSetSchema(&pCommitter->dWriter.bData, pCommitter->skmTable.pTSchema); + code = tBlockDataSetSchema(&pCommitter->dWriter.bData, pCommitter->skmTable.pTSchema, pTbData->suid, pTbData->uid); if (code) goto _err; + + // .data merge while (pBlock && pRow) { int32_t c = tBlockCmprFn(pBlock, &(SBlock){.minKey = TSDBROW_KEY(pRow), .maxKey = TSDBROW_KEY(pRow)}); if (c < 0) { // disk @@ -997,25 +984,9 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { } } - // merge with last + // .data append and .last merge code = tsdbMergeCommitLast(pCommitter, pIter); if (code) goto _err; -#if 0 - int32_t nRowLeft = tsdbGetNumOfRowsLessThan(pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = - VERSION_MIN}); if (pCommitter->dReader.pRowInfo) { - for (int32_t iRow = pCommitter->dReader.iRow; iRow < pCommitter->dReader.bDatal.nRow; iRow++) { - int64_t uid = pCommitter->dReader.bDatal.aUid[iRow]; - if (uid == pTbData->uid) { - nRowLeft++; - } - } - } - - while (nRowLeft) { - code = tsdbMergeCommitLast(pCommitter, pIter, &nRowLeft); - if (code) goto _err; - } -#endif // end if (pCommitter->dWriter.mBlock.nItem > 0) { @@ -1081,7 +1052,7 @@ _err: static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { int32_t code = 0; - // data + // .data while (true) { if (pCommitter->dReader.pBlockIdx == NULL || tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) >= 0) break; @@ -1094,61 +1065,64 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { goto _err; } - pCommitter->dReader.iBlockIdx++; - if (pCommitter->dReader.iBlockIdx < taosArrayGetSize(pCommitter->dReader.aBlockIdx)) { - pCommitter->dReader.pBlockIdx = - (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx); - - code = - tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock, NULL); - if (code) goto _err; - } else { - pCommitter->dReader.pBlockIdx = NULL; - } + code = tsdbCommitterNextTableData(pCommitter); + if (code) goto _err; } - // last + // .last while (true) { if (pCommitter->dReader.pRowInfo == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRowInfo, &toTable) >= 0) break; - // commit if not same schema - if (pCommitter->dWriter.bDatal.nRow > 0) { - if (pCommitter->dWriter.bDatal.suid != pCommitter->dReader.pRowInfo->suid || - pCommitter->dWriter.bDatal.suid == 0) { + SBlockData *pBlockDataR = &pCommitter->dReader.bDatal; + SBlockData *pBlockDataW = &pCommitter->dWriter.bDatal; + tb_uid_t suid = pCommitter->dReader.pRowInfo->suid; + tb_uid_t uid = pCommitter->dReader.pRowInfo->uid; + + ASSERT((pBlockDataR->suid && !pBlockDataR->uid) || (!pBlockDataR->suid && pBlockDataR->uid)); + ASSERT(pBlockDataR->nRow > 0); + + // commit and reset block data schema if need + if (pBlockDataW->nRow > 0) { + if (pBlockDataW->suid != pCommitter->dReader.pRowInfo->suid || pBlockDataW->suid == 0) { code = tsdbCommitLastBlock(pCommitter); if (code) goto _err; + + tBlockDataReset(pBlockDataW); } } - if (pCommitter->dWriter.bDatal.nRow == 0) { - code = tsdbCommitterUpdateTableSchema(pCommitter, pCommitter->dReader.pRowInfo->suid, - pCommitter->dReader.pRowInfo->suid, 1 /*TODO*/); + // set block data schema if need + if (pBlockDataW->suid == 0 && pBlockDataW->uid == 0) { + code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid, 1 /*TOOD*/); if (code) goto _err; - pCommitter->dWriter.bDatal.suid = pCommitter->dReader.pRowInfo->suid; - code = tBlockDataSetSchema(&pCommitter->dWriter.bDatal, pCommitter->skmTable.pTSchema); + code = tBlockDataSetSchema(pBlockDataW, pCommitter->skmTable.pTSchema, suid, suid ? 0 : uid); if (code) goto _err; } // check if it can make sure that one table data in one block - int64_t uid = pCommitter->dReader.pRowInfo->uid; int32_t nRow = 0; - for (int32_t iRow = pCommitter->dReader.iRow; - (iRow < pCommitter->dReader.bDatal.nRow) && (pCommitter->dReader.bDatal.aUid[iRow] == uid); iRow++) { - nRow++; + if (pBlockDataR->suid) { + for (int32_t iRow = pCommitter->dReader.iRow; (iRow < pBlockDataR->nRow) && (pBlockDataR->aUid[iRow] == uid); + iRow++) { + nRow++; + } + } else { + ASSERT(pCommitter->dReader.iRow == 0); + nRow = pBlockDataR->nRow; } ASSERT(nRow > 0 && nRow < pCommitter->minRow); - if (pCommitter->dWriter.bDatal.nRow + nRow > pCommitter->maxRow) { - ASSERT(pCommitter->dWriter.bDatal.nRow > 0); + if (pBlockDataW->nRow + nRow > pCommitter->maxRow) { + ASSERT(pBlockDataW->nRow > 0); code = tsdbCommitLastBlock(pCommitter); if (code) goto _err; } while (nRow > 0) { - code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &pCommitter->dReader.pRowInfo->row, NULL); + code = tBlockDataAppendRow(pBlockDataW, &pCommitter->dReader.pRowInfo->row, NULL, uid); if (code) goto _err; code = tsdbCommitterNextLastRow(pCommitter); From 03a615c01adb5575aa4a191955672035cbb3792b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 4 Aug 2022 08:04:34 +0000 Subject: [PATCH 09/92] more --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 32 ++++++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index a04d18aae2..eec7a3e4a6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -41,6 +41,7 @@ struct STsdbSnapReader { SArray* aDelData; // SArray }; +#if 0 static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; STsdb* pTsdb = pReader->pTsdb; @@ -245,9 +246,11 @@ _err: tstrerror(code)); return code; } +#endif int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, STsdbSnapReader** ppReader) { - int32_t code = 0; + int32_t code = 0; +#if 0 STsdbSnapReader* pReader = NULL; // alloc @@ -310,11 +313,13 @@ _err: tsdbError("vgId:%d, vnode snapshot tsdb reader open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, tstrerror(code)); *ppReader = NULL; +#endif return code; } int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { - int32_t code = 0; + int32_t code = 0; +#if 0 STsdbSnapReader* pReader = *ppReader; if (pReader->pDataFReader) { @@ -337,11 +342,13 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { taosMemoryFree(pReader); *ppReader = NULL; +#endif return code; } int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; +#if 0 *ppData = NULL; @@ -380,6 +387,7 @@ _exit: _err: tsdbError("vgId:%d, vnode snapshot tsdb read for %s failed since %s", TD_VID(pReader->pTsdb->pVnode), pReader->pTsdb->path, tstrerror(code)); +#endif return code; } @@ -430,6 +438,7 @@ struct STsdbSnapWriter { SArray* aDelIdxW; }; +#if 0 static int32_t tsdbSnapWriteAppendData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; int32_t iRow = 0; // todo @@ -808,7 +817,8 @@ static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, TABLEID id) { if (code) goto _err; _exit: - tsdbDebug("vgId:%d, vnode snapshot tsdb write data impl for %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path); + tsdbDebug("vgId:%d, vnode snapshot tsdb write data impl for %s", TD_VID(pWriter->pTsdb->pVnode), + pWriter->pTsdb->path); return code; _err: @@ -1036,7 +1046,9 @@ _err: tstrerror(code)); return code; } +#endif +#if 0 static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) { int32_t code = 0; STsdb* pTsdb = pWriter->pTsdb; @@ -1082,9 +1094,11 @@ _err: tstrerror(code)); return code; } +#endif int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter) { - int32_t code = 0; + int32_t code = 0; +#if 0 STsdbSnapWriter* pWriter = NULL; // alloc @@ -1153,11 +1167,13 @@ _err: tsdbError("vgId:%d, tsdb snapshot writer open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, tstrerror(code)); *ppWriter = NULL; +#endif return code; } int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { - int32_t code = 0; + int32_t code = 0; +#if 0 STsdbSnapWriter* pWriter = *ppWriter; if (rollback) { @@ -1188,11 +1204,13 @@ _err: pWriter->pTsdb->path, tstrerror(code)); taosMemoryFree(pWriter); *ppWriter = NULL; +#endif return code; } int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { - int32_t code = 0; + int32_t code = 0; +#if 0 SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; // ts data @@ -1216,10 +1234,12 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) _exit: tsdbDebug("vgId:%d, tsdb snapshot write for %s succeed", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path); + return code; _err: tsdbError("vgId:%d, tsdb snapshot write for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path, tstrerror(code)); +#endif return code; } From 2cb114caad617598790ed0690773d397a9d6003e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 4 Aug 2022 10:15:51 +0000 Subject: [PATCH 10/92] more work --- source/dnode/vnode/src/inc/tsdb.h | 33 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 363 ++++++++++-------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 27 +- 3 files changed, 217 insertions(+), 206 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 4c7dbd0a45..32037cbf19 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -230,16 +230,12 @@ int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter); int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **ppBuf); int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, uint8_t **ppBuf, SBlockIdx *pBlockIdx); int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL, uint8_t **ppBuf); -int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2, - SBlockIdx *pBlockIdx, SBlock *pBlock, int8_t cmprAlg); - -int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); - -/* new */ int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock *pBlock, uint8_t **ppBuf1, uint8_t **ppBuf2, int8_t cmprAlg); int32_t tsdbWriteLastBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockL *pBlockL, uint8_t **ppBuf1, uint8_t **ppBuf2, int8_t cmprAlg); + +int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); // SDataFReader int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFReaderClose(SDataFReader **ppReader); @@ -251,8 +247,6 @@ int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBl int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg, uint8_t **ppBuf); - -/* new */ int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, uint8_t **ppBuf1, @@ -401,12 +395,12 @@ typedef struct { int16_t cid; int8_t type; int8_t smaOn; - int8_t flag; // HAS_NONE|HAS_NULL|HAS_VALUE - int32_t offset; + int8_t flag; // HAS_NONE|HAS_NULL|HAS_VALUE + int32_t szOrigin; // original column value size (only save for variant data type) int32_t szBitmap; // bitmap size int32_t szOffset; // size of offset, only for variant-length data type int32_t szValue; // compressed column value size - int32_t szOrigin; // original column value size (only save for variant data type) + int32_t offset; } SBlockCol; typedef struct { @@ -427,7 +421,6 @@ struct SBlock { int64_t minVersion; int64_t maxVersion; int32_t nRow; - int8_t last; int8_t hasDup; int8_t nSubBlock; SSubBlock aSubBlock[TSDB_MAX_SUBBLOCKS]; @@ -435,26 +428,18 @@ struct SBlock { struct SBlockL { int64_t suid; - struct { - int64_t uid; - int64_t version; - TSKEY ts; - } minKey; - struct { - int64_t uid; - int64_t version; - TSKEY ts; - } maxKey; + int64_t minUid; + int64_t maxUid; int64_t minVer; int64_t maxVer; int32_t nRow; - int8_t cmprAlg; int64_t offset; - int32_t szBlock; + int8_t cmprAlg; int32_t szBlockCol; int32_t szUid; int32_t szVer; int32_t szTSKEY; + int32_t szBlock; }; struct SColData { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 73bac77e6d..2776ac559e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1765,91 +1765,56 @@ static void tsdbUpdateBlockInfo(SBlockData *pBlockData, SBlock *pBlock) { pBlock->nRow += pBlockData->nRow; } -static int32_t tsdbWriteBlockDataKey(SSubBlock *pSubBlock, SBlockData *pBlockData, uint8_t **ppBuf1, int64_t *nDataP, - uint8_t **ppBuf2) { +static int32_t tsdbWriteDataArray(uint8_t *aData, int32_t nEle, int8_t type, int8_t cmprAlg, int32_t *rSize, + uint8_t **ppBuf1, int64_t nBuf1, uint8_t **ppBuf2) { int32_t code = 0; - int64_t size; - int64_t tsize; + int32_t size; - if (pSubBlock->cmprAlg == NO_COMPRESSION) { - pSubBlock->szVersion = sizeof(int64_t) * pSubBlock->nRow; - pSubBlock->szTSKEY = sizeof(TSKEY) * pSubBlock->nRow; - - code = tRealloc(ppBuf1, *nDataP + pSubBlock->szVersion + pSubBlock->szTSKEY + sizeof(TSCKSUM)); - if (code) goto _err; - - // VERSION - memcpy(*ppBuf1 + *nDataP, pBlockData->aVersion, pSubBlock->szVersion); - - // TSKEY - memcpy(*ppBuf1 + *nDataP + pSubBlock->szVersion, pBlockData->aTSKEY, pSubBlock->szTSKEY); + if (IS_VAR_DATA_TYPE(type)) { + size = nEle; } else { - size = (sizeof(int64_t) + sizeof(TSKEY)) * pSubBlock->nRow + COMP_OVERFLOW_BYTES * 2; - - code = tRealloc(ppBuf1, *nDataP + size + sizeof(TSCKSUM)); - if (code) goto _err; - - tsize = sizeof(int64_t) * pSubBlock->nRow + COMP_OVERFLOW_BYTES; - if (pSubBlock->cmprAlg == TWO_STAGE_COMP) { - code = tRealloc(ppBuf2, tsize); - if (code) goto _err; - } - - // VERSION - pSubBlock->szVersion = - tsCompressBigint((char *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, pBlockData->nRow, - *ppBuf1 + *nDataP, size, pSubBlock->cmprAlg, *ppBuf2, tsize); - if (pSubBlock->szVersion <= 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _err; - } - - // TSKEY - pSubBlock->szTSKEY = tsCompressTimestamp((char *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, - pBlockData->nRow, *ppBuf1 + *nDataP + pSubBlock->szVersion, - size - pSubBlock->szVersion, pSubBlock->cmprAlg, *ppBuf2, tsize); - if (pSubBlock->szTSKEY <= 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _err; - } - - ASSERT(pSubBlock->szVersion + pSubBlock->szTSKEY <= size); + size = tDataTypes[type].bytes * nEle; } - // checksum - size = pSubBlock->szVersion + pSubBlock->szTSKEY + sizeof(TSCKSUM); - taosCalcChecksumAppend(0, *ppBuf1 + *nDataP, size); + if (cmprAlg == NO_COMPRESSION) { + code = tRealloc(ppBuf1, nBuf1 + size); + if (code) goto _exit; - *nDataP += size; - return code; + memcpy(*ppBuf1 + nBuf1, aData, size); + *rSize = size; + } else { + code = tRealloc(ppBuf1, size + COMP_OVERFLOW_BYTES); + if (code) goto _exit; -_err: + if (cmprAlg == TWO_STAGE_COMP) { + code = tRealloc(ppBuf2, size + COMP_OVERFLOW_BYTES); + if (code) goto _exit; + } + + int32_t n = tDataTypes[type].compFunc(aData, tDataTypes[type].bytes * nEle, nEle, *ppBuf1 + nBuf1, + size + COMP_OVERFLOW_BYTES, cmprAlg, *ppBuf2, size + COMP_OVERFLOW_BYTES); + if (n <= 0) { + code = TSDB_CODE_COMPRESS_ERROR; + goto _exit; + } + *rSize = n; + } + +_exit: return code; } -static int32_t tsdbWriteColData(SColData *pColData, SBlockCol *pBlockCol, SSubBlock *pSubBlock, uint8_t **ppBuf1, - int64_t *nDataP, uint8_t **ppBuf2) { +static int32_t tsdbWriteColData(SColData *pColData, SBlockCol *pBlockCol, int8_t cmprAlg, uint8_t **ppBuf1, + int64_t nBuf1, uint8_t **ppBuf2) { int32_t code = 0; int64_t size; int64_t n = 0; // BITMAP if (pColData->flag != HAS_VALUE) { - size = BIT2_SIZE(pColData->nVal) + COMP_OVERFLOW_BYTES; - - code = tRealloc(ppBuf1, *nDataP + n + size); + code = tsdbWriteDataArray(pColData->pBitMap, BIT2_SIZE(pColData->nVal), TSDB_DATA_TYPE_TINYINT, cmprAlg, + &pBlockCol->szBitmap, ppBuf1, nBuf1 + n, ppBuf2); if (code) goto _err; - - code = tRealloc(ppBuf2, size); - if (code) goto _err; - - pBlockCol->szBitmap = - tsCompressTinyint((char *)pColData->pBitMap, BIT2_SIZE(pColData->nVal), BIT2_SIZE(pColData->nVal), - *ppBuf1 + *nDataP + n, size, TWO_STAGE_COMP, *ppBuf2, size); - if (pBlockCol->szBitmap <= 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _err; - } } else { pBlockCol->szBitmap = 0; } @@ -1857,60 +1822,29 @@ static int32_t tsdbWriteColData(SColData *pColData, SBlockCol *pBlockCol, SSubBl // OFFSET if (IS_VAR_DATA_TYPE(pColData->type)) { - size = sizeof(int32_t) * pColData->nVal + COMP_OVERFLOW_BYTES; - - code = tRealloc(ppBuf1, *nDataP + n + size); + code = tsdbWriteDataArray((uint8_t *)pColData->aOffset, pColData->nVal, TSDB_DATA_TYPE_INT, cmprAlg, + &pBlockCol->szOffset, ppBuf1, nBuf1 + n, ppBuf2); if (code) goto _err; - - code = tRealloc(ppBuf2, size); - if (code) goto _err; - - pBlockCol->szOffset = tsCompressInt((char *)pColData->aOffset, sizeof(int32_t) * pColData->nVal, pColData->nVal, - *ppBuf1 + *nDataP + n, size, TWO_STAGE_COMP, *ppBuf2, size); - if (pBlockCol->szOffset <= 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _err; - } } else { pBlockCol->szOffset = 0; } n += pBlockCol->szOffset; // VALUE - if (pSubBlock->cmprAlg == NO_COMPRESSION) { - pBlockCol->szValue = pColData->nData; - - code = tRealloc(ppBuf1, *nDataP + n + pBlockCol->szValue + sizeof(TSCKSUM)); + if (pColData->flag != (HAS_NULL | HAS_NONE)) { + code = tsdbWriteDataArray(pColData->pData, pColData->nData, pColData->type, cmprAlg, &pBlockCol->szValue, ppBuf1, + nBuf1 + n, ppBuf2); if (code) goto _err; - - memcpy(*ppBuf1 + *nDataP + n, pColData->pData, pBlockCol->szValue); } else { - size = pColData->nData + COMP_OVERFLOW_BYTES; - - code = tRealloc(ppBuf1, *nDataP + n + size + sizeof(TSCKSUM)); - if (code) goto _err; - - if (pSubBlock->cmprAlg == TWO_STAGE_COMP) { - code = tRealloc(ppBuf2, size); - if (code) goto _err; - } - - pBlockCol->szValue = - tDataTypes[pColData->type].compFunc((char *)pColData->pData, pColData->nData, pColData->nVal, - *ppBuf1 + *nDataP + n, size, pSubBlock->cmprAlg, *ppBuf2, size); - if (pBlockCol->szValue <= 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _err; - } + pBlockCol->szValue = 0; } n += pBlockCol->szValue; - pBlockCol->szOrigin = pColData->nData; // checksum n += sizeof(TSCKSUM); - taosCalcChecksumAppend(0, *ppBuf1 + *nDataP, n); - - *nDataP += n; + code = tRealloc(ppBuf1, nBuf1 + n); + if (code) goto _err; + taosCalcChecksumAppend(0, *ppBuf1 + nBuf1, n); return code; @@ -1918,20 +1852,20 @@ _err: return code; } -static int32_t tsdbWriteBlockDataImpl(TdFilePtr pFD, SSubBlock *pSubBlock, SBlockDataHdr hdr, SArray *aBlockCol, - uint8_t *pData, int64_t nData, uint8_t **ppBuf) { +static int32_t tsdbWriteBlockDataImpl(TdFilePtr pFD, SBlockDataHdr hdr, SArray *aBlockCol, uint8_t *pData, + int64_t nData, uint8_t **ppBuf, int32_t *szBlockCol) { int32_t code = 0; int32_t nBlockCol = taosArrayGetSize(aBlockCol); int64_t size; int64_t n; // HDR + SArray - pSubBlock->szBlockCol = sizeof(hdr); + *szBlockCol = sizeof(hdr); for (int32_t iBlockCol = 0; iBlockCol < nBlockCol; iBlockCol++) { - pSubBlock->szBlockCol += tPutBlockCol(NULL, taosArrayGet(aBlockCol, iBlockCol)); + (*szBlockCol) += tPutBlockCol(NULL, taosArrayGet(aBlockCol, iBlockCol)); } - code = tRealloc(ppBuf, pSubBlock->szBlockCol + sizeof(TSCKSUM)); + code = tRealloc(ppBuf, *szBlockCol + sizeof(TSCKSUM)); if (code) goto _err; n = 0; @@ -1940,11 +1874,11 @@ static int32_t tsdbWriteBlockDataImpl(TdFilePtr pFD, SSubBlock *pSubBlock, SBloc for (int32_t iBlockCol = 0; iBlockCol < nBlockCol; iBlockCol++) { n += tPutBlockCol(*ppBuf + n, taosArrayGet(aBlockCol, iBlockCol)); } - taosCalcChecksumAppend(0, *ppBuf, pSubBlock->szBlockCol + sizeof(TSCKSUM)); + taosCalcChecksumAppend(0, *ppBuf, *szBlockCol + sizeof(TSCKSUM)); - ASSERT(n == pSubBlock->szBlockCol); + ASSERT(n == *szBlockCol); - n = taosWriteFile(pFD, *ppBuf, pSubBlock->szBlockCol + sizeof(TSCKSUM)); + n = taosWriteFile(pFD, *ppBuf, *szBlockCol + sizeof(TSCKSUM)); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -2007,42 +1941,49 @@ _err: return code; } -int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2, - SBlockIdx *pBlockIdx, SBlock *pBlock, int8_t cmprAlg) { - int32_t code = 0; - SSubBlock *pSubBlock = &pBlock->aSubBlock[pBlock->nSubBlock++]; - SBlockCol blockCol; - SBlockCol *pBlockCol = &blockCol; - int64_t n; - TdFilePtr pFileFD = pBlock->last ? pWriter->pLastFD : pWriter->pDataFD; - SBlockDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pBlockIdx->suid, .uid = pBlockIdx->uid}; - uint8_t *p; - int64_t nData; - uint8_t *pBuf1 = NULL; - uint8_t *pBuf2 = NULL; - SArray *aBlockCol = NULL; +int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock *pBlock, uint8_t **ppBuf1, + uint8_t **ppBuf2, int8_t cmprAlg) { + int32_t code = 0; + + ASSERT((pBlockData->suid && pBlockData->uid) || (!pBlockData->suid && pBlockData->uid)); + + uint8_t *pBuf1 = NULL; + uint8_t *pBuf2 = NULL; if (!ppBuf1) ppBuf1 = &pBuf1; if (!ppBuf2) ppBuf2 = &pBuf2; tsdbUpdateBlockInfo(pBlockData, pBlock); + SSubBlock *pSubBlock = &pBlock->aSubBlock[pBlock->nSubBlock++]; + pSubBlock->nRow = pBlockData->nRow; pSubBlock->cmprAlg = cmprAlg; - if (pBlock->last) { - pSubBlock->offset = pWriter->fLast.size; - } else { - pSubBlock->offset = pWriter->fData.size; - } + pSubBlock->offset = pWriter->fData.size; // ======================= BLOCK DATA ======================= - // TSDBKEY - nData = 0; - code = tsdbWriteBlockDataKey(pSubBlock, pBlockData, ppBuf1, &nData, ppBuf2); + int64_t nBuf1 = 0; + + // VERSION + code = tsdbWriteDataArray((uint8_t *)pBlockData->aVersion, pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, + &pSubBlock->szVersion, ppBuf1, nBuf1, ppBuf2); if (code) goto _err; + nBuf1 += pSubBlock->szVersion; + + // TSKEY + code = tsdbWriteDataArray((uint8_t *)pBlockData->aTSKEY, pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg, + &pSubBlock->szTSKEY, ppBuf1, nBuf1, ppBuf2); + if (code) goto _err; + nBuf1 += pSubBlock->szTSKEY; + + // checksum + nBuf1 += sizeof(TSCKSUM); + code = tRealloc(ppBuf1, nBuf1); + if (code) goto _err; + taosCalcChecksumAppend(0, *ppBuf1, nBuf1); // COLUMNS - aBlockCol = taosArrayInit(taosArrayGetSize(pBlockData->aIdx), sizeof(SBlockCol)); + SArray *aBlockCol = taosArrayInit(taosArrayGetSize(pBlockData->aIdx), sizeof(SBlockCol)); if (aBlockCol == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -2050,46 +1991,46 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_ int32_t offset = 0; for (int32_t iCol = 0; iCol < taosArrayGetSize(pBlockData->aIdx); iCol++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iCol); + SBlockCol blockCol = {0}; ASSERT(pColData->flag); if (pColData->flag == HAS_NONE) continue; - pBlockCol->cid = pColData->cid; - pBlockCol->type = pColData->type; - pBlockCol->smaOn = pColData->smaOn; - pBlockCol->flag = pColData->flag; + blockCol.cid = pColData->cid; + blockCol.type = pColData->type; + blockCol.smaOn = pColData->smaOn; + blockCol.flag = pColData->flag; + blockCol.szOrigin = pColData->nData; if (pColData->flag != HAS_NULL) { - code = tsdbWriteColData(pColData, pBlockCol, pSubBlock, ppBuf1, &nData, ppBuf2); + code = tsdbWriteColData(pColData, &blockCol, cmprAlg, ppBuf1, nBuf1, ppBuf2); if (code) goto _err; - pBlockCol->offset = offset; - offset = offset + pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM); + blockCol.offset = offset; + offset = offset + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); + nBuf1 = nBuf1 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); } - if (taosArrayPush(aBlockCol, pBlockCol) == NULL) { + if (taosArrayPush(aBlockCol, &blockCol) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } } // write - code = tsdbWriteBlockDataImpl(pFileFD, pSubBlock, hdr, aBlockCol, *ppBuf1, nData, ppBuf2); + SBlockDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pBlockData->suid, .uid = pBlockData->uid}; + code = tsdbWriteBlockDataImpl(pWriter->pDataFD, hdr, aBlockCol, *ppBuf1, nBuf1, ppBuf2, &pSubBlock->szBlockCol); if (code) goto _err; - pSubBlock->szBlock = pSubBlock->szBlockCol + sizeof(TSCKSUM) + nData; - if (pBlock->last) { - pWriter->fLast.size += pSubBlock->szBlock; - } else { - pWriter->fData.size += pSubBlock->szBlock; - } + pSubBlock->szBlock = pSubBlock->szBlockCol + sizeof(TSCKSUM) + nBuf1; + pWriter->fData.size += pSubBlock->szBlock; // ======================= BLOCK SMA ======================= pSubBlock->sOffset = 0; pSubBlock->nSma = 0; - if (pBlock->nSubBlock > 1 || pBlock->last || pBlock->hasDup) goto _exit; + if (pBlock->nSubBlock > 1 || pBlock->hasDup) goto _exit; code = tsdbWriteBlockSma(pWriter->pSmaFD, pBlockData, pSubBlock, ppBuf1); if (code) goto _err; @@ -2113,17 +2054,113 @@ _err: return code; } -int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock *pBlock, uint8_t **ppBuf1, - uint8_t **ppBuf2, int8_t cmprAlg) { - int32_t code = 0; - // TODO - return code; -} - int32_t tsdbWriteLastBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockL *pBlockL, uint8_t **ppBuf1, uint8_t **ppBuf2, int8_t cmprAlg) { int32_t code = 0; - // TODO + + ASSERT((pBlockData->suid && !pBlockData->uid) || (!pBlockData->suid && pBlockData->uid)); + + uint8_t *pBuf1 = NULL; + uint8_t *pBuf2 = NULL; + + if (!ppBuf1) ppBuf1 = &pBuf1; + if (!ppBuf2) ppBuf2 = &pBuf2; + + pBlockL->suid = pBlockData->suid; + if (pBlockData->uid) { + pBlockL->maxUid = pBlockL->minUid = pBlockData->uid; + } else { + pBlockL->minUid = pBlockData->aUid[0]; + pBlockL->maxUid = pBlockData->aUid[pBlockData->nRow - 1]; + } + pBlockL->nRow = pBlockData->nRow; + pBlockL->offset = pWriter->fLast.size; + pBlockL->cmprAlg = cmprAlg; + pBlockL->minVer = VERSION_MAX; + pBlockL->maxVer = VERSION_MIN; + for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { + pBlockL->minVer = TMIN(pBlockL->minVer, pBlockData->aVersion[iRow]); + pBlockL->maxVer = TMAX(pBlockL->maxVer, pBlockData->aVersion[iRow]); + } + + // ======================= BLOCK DATA ======================= + int64_t nBuf1 = 0; + + // UID + if (pBlockData->uid == 0) { + code = tsdbWriteDataArray((uint8_t *)pBlockData->aUid, pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, + &pBlockL->szUid, ppBuf1, nBuf1, ppBuf2); + if (code) goto _err; + } else { + pBlockL->szUid = 0; + } + nBuf1 += pBlockL->szUid; + + // VERSION + code = tsdbWriteDataArray((uint8_t *)pBlockData->aVersion, pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, + &pBlockL->szVer, ppBuf1, nBuf1, ppBuf2); + if (code) goto _err; + nBuf1 += pBlockL->szVer; + + // TSKEY + code = tsdbWriteDataArray((uint8_t *)pBlockData->aTSKEY, pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg, + &pBlockL->szTSKEY, ppBuf1, nBuf1, ppBuf2); + if (code) goto _err; + nBuf1 += pBlockL->szTSKEY; + + // checksum + nBuf1 += sizeof(TSCKSUM); + code = tRealloc(ppBuf1, nBuf1); + if (code) goto _err; + taosCalcChecksumAppend(0, *ppBuf1, nBuf1); + + // COLUMNS + SArray *aBlockCol = taosArrayInit(taosArrayGetSize(pBlockData->aIdx), sizeof(SBlockCol)); + if (aBlockCol == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + int32_t offset = 0; + for (int32_t iCol = 0; iCol < taosArrayGetSize(pBlockData->aIdx); iCol++) { + SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iCol); + SBlockCol blockCol = {0}; + + ASSERT(pColData->flag); + + if (pColData->flag == HAS_NONE) continue; + + blockCol.cid = pColData->cid; + blockCol.type = pColData->type; + blockCol.smaOn = pColData->smaOn; + blockCol.flag = pColData->flag; + blockCol.szOrigin = pColData->nData; + + if (pColData->flag != HAS_NULL) { + code = tsdbWriteColData(pColData, &blockCol, cmprAlg, ppBuf1, nBuf1, ppBuf2); + if (code) goto _err; + + blockCol.offset = offset; + offset = offset + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); + nBuf1 = nBuf1 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); + } + + if (taosArrayPush(aBlockCol, &blockCol) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + } + + // write + SBlockDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pBlockData->suid, .uid = pBlockData->uid}; + code = tsdbWriteBlockDataImpl(pWriter->pLastFD, hdr, aBlockCol, *ppBuf1, nBuf1, ppBuf2, &pBlockL->szBlockCol); + if (code) goto _err; + + pBlockL->szBlock = pBlockL->szBlockCol + sizeof(TSCKSUM) + nBuf1; + pWriter->fLast.size += pBlockL->szBlock; + + return code; + +_err: return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 7beac23912..c17a7f4a09 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -230,7 +230,6 @@ int32_t tPutBlock(uint8_t *p, void *ph) { n += tPutI64v(p ? p + n : p, pBlock->minVersion); n += tPutI64v(p ? p + n : p, pBlock->maxVersion); n += tPutI32v(p ? p + n : p, pBlock->nRow); - n += tPutI8(p ? p + n : p, pBlock->last); n += tPutI8(p ? p + n : p, pBlock->hasDup); n += tPutI8(p ? p + n : p, pBlock->nSubBlock); for (int8_t iSubBlock = 0; iSubBlock < pBlock->nSubBlock; iSubBlock++) { @@ -257,7 +256,6 @@ int32_t tGetBlock(uint8_t *p, void *ph) { n += tGetI64v(p + n, &pBlock->minVersion); n += tGetI64v(p + n, &pBlock->maxVersion); n += tGetI32v(p + n, &pBlock->nRow); - n += tGetI8(p + n, &pBlock->last); n += tGetI8(p + n, &pBlock->hasDup); n += tGetI8(p + n, &pBlock->nSubBlock); for (int8_t iSubBlock = 0; iSubBlock < pBlock->nSubBlock; iSubBlock++) { @@ -290,7 +288,6 @@ int32_t tBlockCmprFn(const void *p1, const void *p2) { bool tBlockHasSma(SBlock *pBlock) { if (pBlock->nSubBlock > 1) return false; - if (pBlock->last) return false; if (pBlock->hasDup) return false; return pBlock->aSubBlock[0].nSma > 0; @@ -301,22 +298,18 @@ int32_t tPutBlockL(uint8_t *p, void *ph) { SBlockL *pBlockL = (SBlockL *)ph; n += tPutI64(p ? p + n : p, pBlockL->suid); - n += tPutI64(p ? p + n : p, pBlockL->minKey.uid); - n += tPutI64v(p ? p + n : p, pBlockL->minKey.version); - n += tPutI64(p ? p + n : p, pBlockL->minKey.ts); - n += tPutI64(p ? p + n : p, pBlockL->maxKey.uid); - n += tPutI64v(p ? p + n : p, pBlockL->maxKey.version); - n += tPutI64(p ? p + n : p, pBlockL->maxKey.ts); + n += tPutI64(p ? p + n : p, pBlockL->minUid); + n += tPutI64(p ? p + n : p, pBlockL->maxUid); n += tPutI64v(p ? p + n : p, pBlockL->minVer); n += tPutI64v(p ? p + n : p, pBlockL->maxVer); n += tPutI32v(p ? p + n : p, pBlockL->nRow); - n += tPutI8(p ? p + n : p, pBlockL->cmprAlg); n += tPutI64v(p ? p + n : p, pBlockL->offset); - n += tPutI32v(p ? p + n : p, pBlockL->szBlock); + n += tPutI8(p ? p + n : p, pBlockL->cmprAlg); n += tPutI32v(p ? p + n : p, pBlockL->szBlockCol); n += tPutI32v(p ? p + n : p, pBlockL->szUid); n += tPutI32v(p ? p + n : p, pBlockL->szVer); n += tPutI32v(p ? p + n : p, pBlockL->szTSKEY); + n += tPutI32v(p ? p + n : p, pBlockL->szBlock); return n; } @@ -326,22 +319,18 @@ int32_t tGetBlockL(uint8_t *p, void *ph) { SBlockL *pBlockL = (SBlockL *)ph; n += tGetI64(p + n, &pBlockL->suid); - n += tGetI64(p + n, &pBlockL->minKey.uid); - n += tGetI64v(p + n, &pBlockL->minKey.version); - n += tGetI64(p + n, &pBlockL->minKey.ts); - n += tGetI64(p + n, &pBlockL->maxKey.uid); - n += tGetI64v(p + n, &pBlockL->maxKey.version); - n += tGetI64(p + n, &pBlockL->maxKey.ts); + n += tGetI64(p + n, &pBlockL->minUid); + n += tGetI64(p + n, &pBlockL->maxUid); n += tGetI64v(p + n, &pBlockL->minVer); n += tGetI64v(p + n, &pBlockL->maxVer); n += tGetI32v(p + n, &pBlockL->nRow); - n += tGetI8(p + n, &pBlockL->cmprAlg); n += tGetI64v(p + n, &pBlockL->offset); - n += tGetI32v(p + n, &pBlockL->szBlock); + n += tGetI8(p + n, &pBlockL->cmprAlg); n += tGetI32v(p + n, &pBlockL->szBlockCol); n += tGetI32v(p + n, &pBlockL->szUid); n += tGetI32v(p + n, &pBlockL->szVer); n += tGetI32v(p + n, &pBlockL->szTSKEY); + n += tGetI32v(p + n, &pBlockL->szBlock); return n; } From d5c2d5941f4bf392225d18318b0115a3039cc557 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 5 Aug 2022 05:23:50 +0000 Subject: [PATCH 11/92] more work --- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 234 +++++++++++++++--- source/dnode/vnode/src/tsdb/tsdbUtil.c | 8 +- 2 files changed, 197 insertions(+), 45 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 2776ac559e..bc4656278c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -729,21 +729,6 @@ _err: return code; } -int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, uint8_t **ppBuf1, - uint8_t **ppBuf2) { - int32_t code = 0; - // TODO - return code; -} - -int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, uint8_t **ppBuf1, - uint8_t **ppBuf2) { - int32_t code = 0; - ASSERT(0); - // TODO - return code; -} - static int32_t tsdbReadBlockDataKey(SBlockData *pBlockData, SSubBlock *pSubBlock, uint8_t *pBuf, uint8_t **ppBuf) { int32_t code = 0; int64_t size = pSubBlock->szVersion + pSubBlock->szTSKEY + sizeof(TSCKSUM); @@ -892,20 +877,25 @@ _err: return code; } -static int32_t tsdbReadBlockCol(SSubBlock *pSubBlock, uint8_t *p, SArray *aBlockCol) { +static int32_t tsdbReadBlockCol(uint8_t *pBuf, int32_t szBlockCol, SBlockDataHdr *pHdr, SArray *aBlockCol) { int32_t code = 0; int32_t n = 0; SBlockCol blockCol; SBlockCol *pBlockCol = &blockCol; - if (!taosCheckChecksumWhole(p, pSubBlock->szBlockCol + sizeof(TSCKSUM))) { + // checksum + if (!taosCheckChecksumWhole(pBuf, szBlockCol + sizeof(TSCKSUM))) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } + // hdr + *pHdr = *(SBlockDataHdr *)pBuf; n += sizeof(SBlockDataHdr); - while (n < pSubBlock->szBlockCol) { - n += tGetBlockCol(p + n, pBlockCol); + + // aBlockCol + while (n < szBlockCol) { + n += tGetBlockCol(pBuf + n, pBlockCol); if (taosArrayPush(aBlockCol, pBlockCol) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -913,7 +903,7 @@ static int32_t tsdbReadBlockCol(SSubBlock *pSubBlock, uint8_t *p, SArray *aBlock } } - ASSERT(n == pSubBlock->szBlockCol); + ASSERT(n == szBlockCol); return code; @@ -921,10 +911,48 @@ _err: return code; } +static int32_t tsdbReadDataArray(uint8_t *pInput, int32_t szInput, int32_t nEle, int8_t type, int8_t cmprAlg, + uint8_t **ppOut, uint8_t **ppBuf) { + int32_t code = 0; + int32_t size; + + // size + if (IS_VAR_DATA_TYPE(type)) { + size = nEle; + } else { + size = tDataTypes[type].bytes * nEle; + } + + // alloc + code = tRealloc(ppOut, size); + if (code) goto _exit; + + // decode + if (cmprAlg == NO_COMPRESSION) { + ASSERT(szInput == size); + memcpy(*ppOut, pInput, size); + } else { + if (cmprAlg == TWO_STAGE_COMP) { + code = tRealloc(ppBuf, size + COMP_OVERFLOW_BYTES); + if (code) goto _exit; + + int32_t n = + tDataTypes[type].decompFunc(pInput, szInput, nEle, *ppOut, size, cmprAlg, *ppBuf, size + COMP_OVERFLOW_BYTES); + if (n <= 0) { + code = TSDB_CODE_COMPRESS_ERROR; + goto _exit; + } + } + } + +_exit: + return code; +} + static int32_t tsdbReadSubColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int32_t iSubBlock, int16_t *aColId, int32_t nCol, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2) { - TdFilePtr pFD = pBlock->last ? pReader->pLastFD : pReader->pDataFD; + TdFilePtr pFD = pReader->pDataFD; SSubBlock *pSubBlock = &pBlock->aSubBlock[iSubBlock]; SArray *aBlockCol = NULL; int32_t code = 0; @@ -974,7 +1002,7 @@ static int32_t tsdbReadSubColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, S goto _err; } - code = tsdbReadBlockCol(pSubBlock, *ppBuf1, aBlockCol); + code = tsdbReadBlockCol(*ppBuf1, pSubBlock->szBlock, NULL /*todo*/, aBlockCol); if (code) goto _err; code = tsdbReadBlockDataKey(pBlockData, pSubBlock, *ppBuf1 + pSubBlock->szBlockCol + sizeof(TSCKSUM), ppBuf2); @@ -1093,13 +1121,13 @@ _err: return code; } -static int32_t tsdbReadSubBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int32_t iSubBlock, - SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2) { +static int32_t tsdbReadSubBlockData(SDataFReader *pReader, SBlock *pBlock, int32_t iSubBlock, SBlockData *pBlockData, + uint8_t **ppBuf1, uint8_t **ppBuf2) { int32_t code = 0; uint8_t *p; int64_t size; int64_t n; - TdFilePtr pFD = pBlock->last ? pReader->pLastFD : pReader->pDataFD; + TdFilePtr pFD = pReader->pDataFD; SSubBlock *pSubBlock = &pBlock->aSubBlock[iSubBlock]; SArray *aBlockCol = NULL; @@ -1175,20 +1203,18 @@ _err: return code; } -int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, SBlockData *pBlockData, - uint8_t **ppBuf1, uint8_t **ppBuf2) { - int32_t code = 0; - TdFilePtr pFD = pBlock->last ? pReader->pLastFD : pReader->pDataFD; - uint8_t *pBuf1 = NULL; - uint8_t *pBuf2 = NULL; - int32_t iSubBlock; +int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, uint8_t **ppBuf1, + uint8_t **ppBuf2) { + int32_t code = 0; + uint8_t *pBuf1 = NULL; + uint8_t *pBuf2 = NULL; if (!ppBuf1) ppBuf1 = &pBuf1; if (!ppBuf2) ppBuf2 = &pBuf2; // read the first sub-block - iSubBlock = 0; - code = tsdbReadSubBlockData(pReader, pBlockIdx, pBlock, iSubBlock, pBlockData, ppBuf1, ppBuf2); + int32_t iSubBlock = 0; + code = tsdbReadSubBlockData(pReader, pBlock, iSubBlock, pBlockData, ppBuf1, ppBuf2); if (code) goto _err; // read remain block data and do merg @@ -1199,7 +1225,7 @@ int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *p tBlockDataInit(pBlockData1); tBlockDataInit(pBlockData2); for (iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { - code = tsdbReadSubBlockData(pReader, pBlockIdx, pBlock, iSubBlock, pBlockData1, ppBuf1, ppBuf2); + code = tsdbReadSubBlockData(pReader, pBlock, iSubBlock, pBlockData1, ppBuf1, ppBuf2); if (code) { tBlockDataClear(pBlockData1, 1); tBlockDataClear(pBlockData2, 1); @@ -1230,14 +1256,138 @@ int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *p ASSERT(tsdbKeyCmprFn(&pBlock->minKey, &TSDBROW_KEY(&tBlockDataFirstRow(pBlockData))) == 0); ASSERT(tsdbKeyCmprFn(&pBlock->maxKey, &TSDBROW_KEY(&tBlockDataLastRow(pBlockData))) == 0); - if (pBuf1) tFree(pBuf1); - if (pBuf2) tFree(pBuf2); + tFree(pBuf1); + tFree(pBuf2); return code; _err: tsdbError("vgId:%d, tsdb read block data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); - if (pBuf1) tFree(pBuf1); - if (pBuf2) tFree(pBuf2); + tFree(pBuf1); + tFree(pBuf2); + return code; +} + +int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, uint8_t **ppBuf1, + uint8_t **ppBuf2) { + int32_t code = 0; + + tBlockDataReset(pBlockData); + + uint8_t *pBuf1 = NULL; + uint8_t *pBuf2 = NULL; + if (!ppBuf1) ppBuf1 = &pBuf1; + if (!ppBuf2) ppBuf2 = &pBuf2; + + // realloc + code = tRealloc(ppBuf1, pBlockL->szBlock); + if (code) goto _err; + + // seek + int64_t n = taosLSeekFile(pReader->pLastFD, pBlockL->offset, SEEK_SET); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // read + n = taosReadFile(pReader->pLastFD, *ppBuf1, pBlockL->szBlock); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } else if (n < pBlockL->szBlock) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + + // decode block col + SBlockDataHdr hdr; + SArray *aBlockCol = taosArrayInit(0, sizeof(SBlockCol)); + uint8_t *p = *ppBuf1; + if (aBlockCol == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + code = tsdbReadBlockCol(p, pBlockL->szBlockCol, &hdr, aBlockCol); + if (code) goto _err; + p += pBlockL->szBlockCol + sizeof(TSCKSUM); + + // checksum + if (!taosCheckChecksumWhole(p, pBlockL->szUid + pBlockL->szVer + pBlockL->szTSKEY + sizeof(TSCKSUM))) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + + // UID + if (hdr.uid == 0) { + code = tsdbReadDataArray(p, pBlockL->szUid, pBlockL->nRow, TSDB_DATA_TYPE_BIGINT, pBlockL->cmprAlg, + (uint8_t **)&pBlockData->aUid, ppBuf2); + if (code) goto _err; + } else { + ASSERT(pBlockL->szUid == 0); + } + p += pBlockL->szUid; + + // VERSION + code = tsdbReadDataArray(p, pBlockL->szVer, pBlockL->nRow, TSDB_DATA_TYPE_BIGINT, pBlockL->cmprAlg, + (uint8_t **)&pBlockData->aVersion, ppBuf2); + if (code) goto _err; + p += pBlockL->szVer; + + // TSKEY + code = tsdbReadDataArray(p, pBlockL->szTSKEY, pBlockL->nRow, TSDB_DATA_TYPE_TIMESTAMP, pBlockL->cmprAlg, + (uint8_t **)&pBlockData->aTSKEY, ppBuf2); + if (code) goto _err; + p += pBlockL->szTSKEY; + p += sizeof(TSCKSUM); + + // COLUMN + code = tBlockDataSetSchema(pBlockData, NULL, hdr.suid, hdr.uid); + if (code) goto _err; + + for (int32_t iBlockCol = 0; iBlockCol < taosArrayGetSize(aBlockCol); iBlockCol++) { + SBlockCol *pBlockCol = (SBlockCol *)taosArrayGet(aBlockCol, iBlockCol); + SColData *pColData; + + // checksum + if (!taosCheckChecksumWhole(p, pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM))) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + + // add SColData + code = tBlockDataAddColData(pBlockData, iBlockCol, &pColData); + if (code) goto _err; + tColDataInit(pColData, pBlockCol->cid, pBlockCol->type, pBlockCol->smaOn); + pColData->nVal = pBlockL->nRow; + pColData->flag = pBlockCol->flag; + + // bitmap + if (pBlockCol->szBitmap) { + code = tsdbReadDataArray(p, pBlockCol->szBitmap, ); + if (code) goto _err; + } + p += pBlockCol->szBitmap; + + // offset + if (pBlockCol->szOffset) { + code = tsdbReadDataArray(p, pBlockCol->szOffset, ); + if (code) goto _err; + } + p += pBlockCol->szOffset; + + // value + pColData->nData = pBlockCol->szOrigin; + if (pColData->nData) { + // TODO + } + } + + taosArrayDestroy(aBlockCol); + + return code; + +_err: + tsdbError("vgId:%d tsdb read last block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; } @@ -1832,10 +1982,12 @@ static int32_t tsdbWriteColData(SColData *pColData, SBlockCol *pBlockCol, int8_t // VALUE if (pColData->flag != (HAS_NULL | HAS_NONE)) { + ASSERT(pColData->nData); code = tsdbWriteDataArray(pColData->pData, pColData->nData, pColData->type, cmprAlg, &pBlockCol->szValue, ppBuf1, nBuf1 + n, ppBuf2); if (code) goto _err; } else { + ASSERT(pColData->nData == 0); pBlockCol->szValue = 0; } n += pBlockCol->szValue; @@ -2073,11 +2225,11 @@ int32_t tsdbWriteLastBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock pBlockL->minUid = pBlockData->aUid[0]; pBlockL->maxUid = pBlockData->aUid[pBlockData->nRow - 1]; } + pBlockL->minVer = VERSION_MAX; + pBlockL->maxVer = VERSION_MIN; pBlockL->nRow = pBlockData->nRow; pBlockL->offset = pWriter->fLast.size; pBlockL->cmprAlg = cmprAlg; - pBlockL->minVer = VERSION_MAX; - pBlockL->maxVer = VERSION_MIN; for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { pBlockL->minVer = TMIN(pBlockL->minVer, pBlockData->aVersion[iRow]); pBlockL->maxVer = TMAX(pBlockL->maxVer, pBlockData->aVersion[iRow]); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index c17a7f4a09..1ae241f0c1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -348,11 +348,11 @@ int32_t tPutBlockCol(uint8_t *p, void *ph) { n += tPutI8(p ? p + n : p, pBlockCol->flag); if (pBlockCol->flag != HAS_NULL) { - n += tPutI32v(p ? p + n : p, pBlockCol->offset); + n += tPutI32v(p ? p + n : p, pBlockCol->szOrigin); n += tPutI32v(p ? p + n : p, pBlockCol->szBitmap); n += tPutI32v(p ? p + n : p, pBlockCol->szOffset); n += tPutI32v(p ? p + n : p, pBlockCol->szValue); - n += tPutI32v(p ? p + n : p, pBlockCol->szOrigin); + n += tPutI32v(p ? p + n : p, pBlockCol->offset); } return n; @@ -370,11 +370,11 @@ int32_t tGetBlockCol(uint8_t *p, void *ph) { ASSERT(pBlockCol->flag && (pBlockCol->flag != HAS_NONE)); if (pBlockCol->flag != HAS_NULL) { - n += tGetI32v(p + n, &pBlockCol->offset); + n += tGetI32v(p + n, &pBlockCol->szOrigin); n += tGetI32v(p + n, &pBlockCol->szBitmap); n += tGetI32v(p + n, &pBlockCol->szOffset); n += tGetI32v(p + n, &pBlockCol->szValue); - n += tGetI32v(p + n, &pBlockCol->szOrigin); + n += tGetI32v(p + n, &pBlockCol->offset); } return n; From 51538a1710ba247f0cfd29e64a463342164b5fe1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 5 Aug 2022 09:47:26 +0000 Subject: [PATCH 12/92] more work --- source/common/src/ttypes.c | 4 +- source/dnode/vnode/src/inc/tsdb.h | 76 ++++--- source/dnode/vnode/src/tsdb/tsdbRead.c | 24 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 10 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 208 +++++++++++++++++- 5 files changed, 271 insertions(+), 51 deletions(-) diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 156b66ae86..fee89e2f37 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -392,10 +392,10 @@ tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = { getStatics_i64}, {TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", 0, 0, tsCompressFloat, tsDecompressFloat, getStatics_f}, {TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", 0, 0, tsCompressDouble, tsDecompressDouble, getStatics_d}, - {TSDB_DATA_TYPE_VARCHAR, 6, 0, "VARCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_bin}, + {TSDB_DATA_TYPE_VARCHAR, 6, 1, "VARCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_bin}, {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", INT64_MIN, INT64_MAX, tsCompressTimestamp, tsDecompressTimestamp, getStatics_i64}, - {TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr}, + {TSDB_DATA_TYPE_NCHAR, 5, 1, "NCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr}, {TSDB_DATA_TYPE_UTINYINT, 16, CHAR_BYTES, "TINYINT UNSIGNED", 0, UINT8_MAX, tsCompressTinyint, tsDecompressTinyint, getStatics_u8}, {TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES, "SMALLINT UNSIGNED", 0, UINT16_MAX, tsCompressSmallint, diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 32037cbf19..72296e4551 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -47,6 +47,7 @@ typedef struct SBlockL SBlockL; typedef struct SColData SColData; typedef struct SBlockDataHdr SBlockDataHdr; typedef struct SBlockData SBlockData; +typedef struct SDiskData SDiskData; typedef struct SDelFile SDelFile; typedef struct SHeadFile SHeadFile; typedef struct SDataFile SDataFile; @@ -149,6 +150,11 @@ SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx); void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData); int32_t tPutBlockData(uint8_t *p, SBlockData *pBlockData); int32_t tGetBlockData(uint8_t *p, SBlockData *pBlockData); +// SDiskData +int32_t tDiskDataInit(SDiskData *pDiskData); +void tDiskDataClear(SDiskData *pDiskData); +int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cmprAlg); +int32_t tDiskToBlockData(SDiskData *pDiskData, SBlockData *pBlockData); // SDelIdx int32_t tPutDelIdx(uint8_t *p, void *ph); int32_t tGetDelIdx(uint8_t *p, void *ph); @@ -392,38 +398,35 @@ struct SMapData { }; typedef struct { - int16_t cid; - int8_t type; - int8_t smaOn; - int8_t flag; // HAS_NONE|HAS_NULL|HAS_VALUE - int32_t szOrigin; // original column value size (only save for variant data type) - int32_t szBitmap; // bitmap size - int32_t szOffset; // size of offset, only for variant-length data type - int32_t szValue; // compressed column value size - int32_t offset; + int16_t cid; + int8_t type; + int8_t smaOn; + int8_t flag; // HAS_NONE|HAS_NULL|HAS_VALUE + int32_t szOrigin; // original column value size (only save for variant data type) + int32_t szBitmap; // bitmap size + int32_t szOffset; // size of offset, only for variant-length data type + int32_t szValue; // compressed column value size + int32_t offset; + uint8_t **ppData; } SBlockCol; typedef struct { - int32_t nRow; - int8_t cmprAlg; - int64_t offset; // block data offset - int32_t szBlockCol; // SBlockCol size - int32_t szVersion; // VERSION size - int32_t szTSKEY; // TSKEY size - int32_t szBlock; // total block size - int64_t sOffset; // sma offset - int32_t nSma; // sma size + int64_t offset; // block data offset + int32_t szBlock; + int32_t szKey; } SSubBlock; struct SBlock { TSDBKEY minKey; TSDBKEY maxKey; - int64_t minVersion; - int64_t maxVersion; + int64_t minVer; + int64_t maxVer; int32_t nRow; int8_t hasDup; int8_t nSubBlock; SSubBlock aSubBlock[TSDB_MAX_SUBBLOCKS]; + int64_t sOffset; // sma offset + int32_t nSma; // sma size }; struct SBlockL { @@ -433,13 +436,6 @@ struct SBlockL { int64_t minVer; int64_t maxVer; int32_t nRow; - int64_t offset; - int8_t cmprAlg; - int32_t szBlockCol; - int32_t szUid; - int32_t szVer; - int32_t szTSKEY; - int32_t szBlock; }; struct SColData { @@ -502,13 +498,17 @@ struct SDelIdx { int64_t size; }; -#pragma pack(push, 1) struct SBlockDataHdr { uint32_t delimiter; + int32_t nRow; int64_t suid; int64_t uid; + int32_t szUid; + int32_t szVer; + int32_t szKey; + int32_t szBlkCol; + int8_t cmprAlg; }; -#pragma pack(pop) struct SDelFile { volatile int32_t nRef; @@ -596,6 +596,24 @@ struct STsdbReadSnap { STsdbFS fs; }; +struct SDiskData { + int8_t cmprAlg; + int32_t nRow; + int64_t suid; + int64_t uid; + int32_t szUid; + int32_t szVer; + int32_t szKey; + + uint8_t *pUid; + uint8_t *pVer; + uint8_t *pKey; + SArray *aBlockCol; // SArray + int32_t nBuf; + SArray *aBuf; // SArray + uint8_t *pBuf; +}; + // ========== inline functions ========== static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { TSDBKEY *pKey1 = (TSDBKEY *)p1; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index cadb6fefd2..84ecaa605d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -630,7 +630,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, uint32_ } // 2. version range check - if (block.minVersion > pReader->verRange.maxVer || block.maxVersion < pReader->verRange.minVer) { + if (block.minVer > pReader->verRange.maxVer || block.maxVer < pReader->verRange.minVer) { continue; } @@ -766,7 +766,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn tsdbDebug("%p copy file block to sdatablock, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, remain:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s", pReader, pBlockIter->index, pFBlock->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, remain, unDumpedRows, - pBlock->minVersion, pBlock->maxVersion, elapsedTime, pReader->idStr); + pBlock->minVer, pBlock->maxVer, elapsedTime, pReader->idStr); return TSDB_CODE_SUCCESS; } @@ -798,7 +798,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI tsdbDebug("%p load file block into buffer, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s", pReader, pBlockIter->index, pFBlock->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow, - pBlock->minVersion, pBlock->maxVersion, elapsedTime, pReader->idStr); + pBlock->minVer, pBlock->maxVer, elapsedTime, pReader->idStr); return TSDB_CODE_SUCCESS; @@ -1011,8 +1011,8 @@ static bool blockIteratorNext(SDataBlockIter* pBlockIter) { static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* pVerRange, SBlock* pBlock) { return (pWindow->ekey < pBlock->maxKey.ts && pWindow->ekey >= pBlock->minKey.ts) || (pWindow->skey > pBlock->minKey.ts && pWindow->skey <= pBlock->maxKey.ts) || - (pVerRange->minVer > pBlock->minVersion && pVerRange->minVer <= pBlock->maxVersion) || - (pVerRange->maxVer < pBlock->maxVersion && pVerRange->maxVer >= pBlock->minVersion); + (pVerRange->minVer > pBlock->minVer && pVerRange->minVer <= pBlock->maxVer) || + (pVerRange->maxVer < pBlock->maxVer && pVerRange->maxVer >= pBlock->minVer); } static SBlock* getNeighborBlockOfSameTable(SFileDataBlockInfo* pFBlockInfo, STableBlockScanInfo* pTableBlockScanInfo, @@ -1092,8 +1092,8 @@ static bool bufferDataInFileBlockGap(int32_t order, TSDBKEY key, SBlock* pBlock) } static bool keyOverlapFileBlock(TSDBKEY key, SBlock* pBlock, SVersionRange* pVerRange) { - return (key.ts >= pBlock->minKey.ts && key.ts <= pBlock->maxKey.ts) && (pBlock->maxVersion >= pVerRange->minVer) && - (pBlock->minVersion <= pVerRange->maxVer); + return (key.ts >= pBlock->minKey.ts && key.ts <= pBlock->maxKey.ts) && (pBlock->maxVer >= pVerRange->minVer) && + (pBlock->minVer <= pVerRange->maxVer); } static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock) { @@ -1102,11 +1102,11 @@ static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, cons for (int32_t i = pBlockScanInfo->fileDelIndex; i < num; i += 1) { TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i); if (p->ts >= pBlock->minKey.ts && p->ts <= pBlock->maxKey.ts) { - if (p->version >= pBlock->minVersion) { + if (p->version >= pBlock->minVer) { return true; } } else if (p->ts < pBlock->minKey.ts) { // p->ts < pBlock->minKey.ts - if (p->version >= pBlock->minVersion) { + if (p->version >= pBlock->minVer) { if (i < num - 1) { TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1); if (i + 1 == num - 1) { // pnext is the last point @@ -1114,7 +1114,7 @@ static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, cons return true; } } else { - if (pnext->ts >= pBlock->minKey.ts && pnext->version >= pBlock->minVersion) { + if (pnext->ts >= pBlock->minKey.ts && pnext->version >= pBlock->minVer) { return true; } } @@ -2512,9 +2512,7 @@ void* tsdbGetIvtIdx(SMeta* pMeta) { return metaGetIvtIdx(pMeta); } -uint64_t getReaderMaxVersion(STsdbReader *pReader) { - return pReader->verRange.maxVer; -} +uint64_t getReaderMaxVersion(STsdbReader* pReader) { return pReader->verRange.maxVer; } /** * @brief Get all suids since suid diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index bc4656278c..eeb11abc97 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1909,8 +1909,8 @@ static void tsdbUpdateBlockInfo(SBlockData *pBlockData, SBlock *pBlock) { pBlock->maxKey = key; } - pBlock->minVersion = TMIN(pBlock->minVersion, key.version); - pBlock->maxVersion = TMAX(pBlock->maxVersion, key.version); + pBlock->minVer = TMIN(pBlock->minVer, key.version); + pBlock->maxVer = TMAX(pBlock->maxVer, key.version); } pBlock->nRow += pBlockData->nRow; } @@ -2310,9 +2310,13 @@ int32_t tsdbWriteLastBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock pBlockL->szBlock = pBlockL->szBlockCol + sizeof(TSCKSUM) + nBuf1; pWriter->fLast.size += pBlockL->szBlock; + tFree(pBuf1); + tFree(pBuf2); return code; _err: + tFree(pBuf1); + tFree(pBuf2); return code; } @@ -2426,4 +2430,4 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { _err: tsdbError("vgId:%d, tsdb DFileSet copy failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 1ae241f0c1..13d293f86b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -227,8 +227,8 @@ int32_t tPutBlock(uint8_t *p, void *ph) { n += tPutTSDBKEY(p ? p + n : p, &pBlock->minKey); n += tPutTSDBKEY(p ? p + n : p, &pBlock->maxKey); - n += tPutI64v(p ? p + n : p, pBlock->minVersion); - n += tPutI64v(p ? p + n : p, pBlock->maxVersion); + n += tPutI64v(p ? p + n : p, pBlock->minVer); + n += tPutI64v(p ? p + n : p, pBlock->maxVer); n += tPutI32v(p ? p + n : p, pBlock->nRow); n += tPutI8(p ? p + n : p, pBlock->hasDup); n += tPutI8(p ? p + n : p, pBlock->nSubBlock); @@ -253,8 +253,8 @@ int32_t tGetBlock(uint8_t *p, void *ph) { n += tGetTSDBKEY(p + n, &pBlock->minKey); n += tGetTSDBKEY(p + n, &pBlock->maxKey); - n += tGetI64v(p + n, &pBlock->minVersion); - n += tGetI64v(p + n, &pBlock->maxVersion); + n += tGetI64v(p + n, &pBlock->minVer); + n += tGetI64v(p + n, &pBlock->maxVer); n += tGetI32v(p + n, &pBlock->nRow); n += tGetI8(p + n, &pBlock->hasDup); n += tGetI8(p + n, &pBlock->nSubBlock); @@ -1509,6 +1509,206 @@ int32_t tGetBlockData(uint8_t *p, SBlockData *pBlockData) { return n; } +// SDiskData ============================== +static int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t nOut, + int32_t *szOut, uint8_t **ppBuf) { + int32_t code = 0; + + ASSERT(szIn > 0 && ppOut); + + if (cmprAlg == NO_COMPRESSION) { + code = tRealloc(ppOut, nOut + szIn); + if (code) goto _exit; + + memcpy(*ppOut + nOut, pIn, szIn); + *szOut = szIn; + } else { + int32_t size = szIn + COMP_OVERFLOW_BYTES; + + code = tRealloc(ppOut, nOut + size); + if (code) goto _exit; + + if (cmprAlg == TWO_STAGE_COMP) { + ASSERT(ppBuf); + code = tRealloc(ppBuf, size); + if (code) goto _exit; + } + + *szOut = + tDataTypes[type].compFunc(pIn, szIn, szIn / tDataTypes[type].bytes, *ppOut + nOut, size, cmprAlg, *ppBuf, size); + if (*szOut <= 0) { + code = TSDB_CODE_COMPRESS_ERROR; + goto _exit; + } + } + +_exit: + return code; +} + +static int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppBuf) { + int32_t code = 0; + + int32_t n = 0; + // bitmap + if (pColData->flag != HAS_VALUE) { + code = tsdbCmprData(pColData->pBitMap, BIT2_SIZE(pColData->nVal), TSDB_DATA_TYPE_TINYINT, cmprAlg, + pBlockCol->ppData, n, &pBlockCol->szBitmap, ppBuf); + if (code) goto _exit; + } else { + pBlockCol->szBitmap = 0; + } + n += pBlockCol->szBitmap; + + // offset + if (IS_VAR_DATA_TYPE(pColData->type)) { + code = tsdbCmprData((uint8_t *)pColData->aOffset, sizeof(int32_t) * pColData->nVal, TSDB_DATA_TYPE_INT, cmprAlg, + pBlockCol->ppData, n, &pBlockCol->szOffset, ppBuf); + if (code) goto _exit; + } else { + pBlockCol->szOffset = 0; + } + n += pBlockCol->szOffset; + + // value + if (pColData->flag != (HAS_NULL | HAS_NONE)) { + code = tsdbCmprData((uint8_t *)pColData->pData, pColData->nData, pColData->type, cmprAlg, pBlockCol->ppData, n, + &pBlockCol->szValue, ppBuf); + if (code) goto _exit; + } else { + pBlockCol->szValue = 0; + } + n += pBlockCol->szValue; + + // checksum + n += sizeof(TSCKSUM); + taosCalcChecksumAppend(0, *ppBuf, n); + +_exit: + return code; +} + +static int32_t tsdbDecmprData() { + int32_t code = 0; + // TODO + return code; +} + +int32_t tDiskDataInit(SDiskData *pDiskData) { + int32_t code = 0; + + pDiskData->aBlockCol = taosArrayInit(0, sizeof(SBlockCol)); + if (pDiskData->aBlockCol == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + pDiskData->aBuf = taosArrayInit(0, sizeof(uint8_t *)); + if (pDiskData->aBuf == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + +_exit: + return code; +} + +void tDiskDataClear(SDiskData *pDiskData) { + taosArrayDestroy(pDiskData->aBlockCol); + for (int32_t i = 0; i < taosArrayGetSize(pDiskData->aBuf); i++) { + tFree((uint8_t *)taosArrayGet(pDiskData->aBuf, i)); + } + taosArrayDestroy(pDiskData->aBuf); +} + +static uint8_t **tDiskDataAllocBuf(SDiskData *pDiskData) { + if (pDiskData->nBuf >= taosArrayGetSize(pDiskData->aBuf)) { + uint8_t *p = NULL; + if (taosArrayPush(pDiskData->aBuf, &p) == NULL) { + return NULL; + } + } + + ASSERT(pDiskData->nBuf < taosArrayGetSize(pDiskData->aBuf)); + uint8_t **pp = taosArrayGet(pDiskData->aBuf, pDiskData->nBuf); + pDiskData->nBuf++; + return pp; +} + +int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cmprAlg) { + int32_t code = 0; + + ASSERT(pBlockData->nRow > 0); + + pDiskData->cmprAlg = cmprAlg; + pDiskData->nRow = pBlockData->nRow; + pDiskData->suid = pBlockData->suid; + pDiskData->uid = pBlockData->uid; + pDiskData->szUid = 0; + pDiskData->szVer = 0; + pDiskData->szKey = 0; + taosArrayClear(pDiskData->aBlockCol); + pDiskData->nBuf = 0; + + // uid + if (pDiskData->uid == 0) { + code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, + &pDiskData->pUid, &pDiskData->szUid, &pDiskData->pBuf); + if (code) goto _exit; + } + + // version + code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, + cmprAlg, &pDiskData->pVer, &pDiskData->szVer, &pDiskData->pBuf); + if (code) goto _exit; + + // tskey + code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, + cmprAlg, &pDiskData->pKey, &pDiskData->szKey, &pDiskData->pBuf); + if (code) goto _exit; + + // columns + int32_t offset = 0; + for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { + SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); + + if (pColData->flag == HAS_NONE) continue; + + SBlockCol blockCol = {.cid = pColData->cid, + .type = pColData->type, + .smaOn = pColData->smaOn, + .flag = pColData->flag, + .szOrigin = pColData->nData}; + + if (pColData->flag != HAS_NULL) { + // alloc a buffer + blockCol.ppData = tDiskDataAllocBuf(pDiskData); + if (blockCol.ppData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + // compress + code = tsdbCmprColData(pColData, cmprAlg, &blockCol); + if (code) goto _exit; + + // update offset + blockCol.offset = offset; + offset = offset + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue; + } + + if (taosArrayPush(pDiskData->aBlockCol, &blockCol) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + } + +_exit: + return code; +} +int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cmprAlg); +int32_t tDiskToBlockData(SDiskData *pDiskData, SBlockData *pBlockData); + // ALGORITHM ============================== void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { SColVal colVal; From 16f556354a979c0b0cb6664c800e5a69fdc6a3e7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 5 Aug 2022 15:22:05 +0000 Subject: [PATCH 13/92] more work --- source/dnode/vnode/src/inc/tsdb.h | 88 ++- source/dnode/vnode/src/tsdb/tsdbCommit.c | 84 ++- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 518 +++++------------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 183 +++++-- 4 files changed, 359 insertions(+), 514 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 72296e4551..08331b1327 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -45,7 +45,7 @@ typedef struct SBlockIdx SBlockIdx; typedef struct SBlock SBlock; typedef struct SBlockL SBlockL; typedef struct SColData SColData; -typedef struct SBlockDataHdr SBlockDataHdr; +typedef struct SDiskDataHdr SDiskDataHdr; typedef struct SBlockData SBlockData; typedef struct SDiskData SDiskData; typedef struct SDelFile SDelFile; @@ -155,6 +155,9 @@ int32_t tDiskDataInit(SDiskData *pDiskData); void tDiskDataClear(SDiskData *pDiskData); int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cmprAlg); int32_t tDiskToBlockData(SDiskData *pDiskData, SBlockData *pBlockData); +// SDiskDataHdr +int32_t tPutDiskDataHdr(uint8_t *p, void *ph); +int32_t tGetDiskDataHdr(uint8_t *p, void *ph); // SDelIdx int32_t tPutDelIdx(uint8_t *p, void *ph); int32_t tGetDelIdx(uint8_t *p, void *ph); @@ -236,10 +239,8 @@ int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter); int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **ppBuf); int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, uint8_t **ppBuf, SBlockIdx *pBlockIdx); int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL, uint8_t **ppBuf); -int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock *pBlock, uint8_t **ppBuf1, - uint8_t **ppBuf2, int8_t cmprAlg); -int32_t tsdbWriteLastBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockL *pBlockL, uint8_t **ppBuf1, - uint8_t **ppBuf2, int8_t cmprAlg); +int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, + int8_t cmprAlg, int8_t toLast, uint8_t **ppBuf); int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); // SDataFReader @@ -403,9 +404,9 @@ typedef struct { int8_t smaOn; int8_t flag; // HAS_NONE|HAS_NULL|HAS_VALUE int32_t szOrigin; // original column value size (only save for variant data type) - int32_t szBitmap; // bitmap size - int32_t szOffset; // size of offset, only for variant-length data type - int32_t szValue; // compressed column value size + int32_t szBitmap; // bitmap size, 0 only for flag == HAS_VAL + int32_t szOffset; // offset size, 0 only for non-variant-length type + int32_t szValue; // value size, 0 when flag == (HAS_NULL | HAS_NONE) int32_t offset; uint8_t **ppData; } SBlockCol; @@ -414,28 +415,33 @@ typedef struct { int64_t offset; // block data offset int32_t szBlock; int32_t szKey; -} SSubBlock; +} SBlockInfo; + +typedef struct { + int64_t offset; + int32_t size; +} SSmaInfo; struct SBlock { - TSDBKEY minKey; - TSDBKEY maxKey; - int64_t minVer; - int64_t maxVer; - int32_t nRow; - int8_t hasDup; - int8_t nSubBlock; - SSubBlock aSubBlock[TSDB_MAX_SUBBLOCKS]; - int64_t sOffset; // sma offset - int32_t nSma; // sma size + TSDBKEY minKey; + TSDBKEY maxKey; + int64_t minVer; + int64_t maxVer; + int32_t nRow; + int8_t hasDup; + int8_t nSubBlock; + SBlockInfo aSubBlock[TSDB_MAX_SUBBLOCKS]; + SSmaInfo smaInfo; }; struct SBlockL { - int64_t suid; - int64_t minUid; - int64_t maxUid; - int64_t minVer; - int64_t maxVer; - int32_t nRow; + int64_t suid; + int64_t minUid; + int64_t maxUid; + int64_t minVer; + int64_t maxVer; + int32_t nRow; + SBlockInfo bInfo; }; struct SColData { @@ -498,15 +504,15 @@ struct SDelIdx { int64_t size; }; -struct SBlockDataHdr { +struct SDiskDataHdr { uint32_t delimiter; - int32_t nRow; int64_t suid; int64_t uid; int32_t szUid; int32_t szVer; int32_t szKey; int32_t szBlkCol; + int32_t nRow; int8_t cmprAlg; }; @@ -575,6 +581,14 @@ struct SDelFWriter { TdFilePtr pWriteH; }; +struct SDiskData { + SDiskDataHdr hdr; + uint8_t **ppKey; + SArray *aBlockCol; // SArray + int32_t nBuf; + SArray *aBuf; // SArray +}; + struct SDataFWriter { STsdb *pTsdb; SDFileSet wSet; @@ -588,6 +602,8 @@ struct SDataFWriter { SDataFile fData; SLastFile fLast; SSmaFile fSma; + + SDiskData dData; }; struct STsdbReadSnap { @@ -596,24 +612,6 @@ struct STsdbReadSnap { STsdbFS fs; }; -struct SDiskData { - int8_t cmprAlg; - int32_t nRow; - int64_t suid; - int64_t uid; - int32_t szUid; - int32_t szVer; - int32_t szKey; - - uint8_t *pUid; - uint8_t *pVer; - uint8_t *pKey; - SArray *aBlockCol; // SArray - int32_t nBuf; - SArray *aBuf; // SArray - uint8_t *pBuf; -}; - // ========== inline functions ========== static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { TSDBKEY *pKey1 = (TSDBKEY *)p1; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index e5c6ee44a0..cd94762c7b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -494,10 +494,11 @@ _exit: } static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) { - int32_t code = 0; - SBlock block; + int32_t code = 0; + SBlockData *pBlockData = &pCommitter->dWriter.bData; + SBlock block; - ASSERT(pCommitter->dWriter.bData.nRow > 0); + ASSERT(pBlockData->nRow > 0); if (pBlock) { block = *pBlock; // as a subblock @@ -505,37 +506,84 @@ static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) { tBlockReset(&block); // as a new block } - code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bData, &block, NULL, NULL, - pCommitter->cmprAlg); - if (code) goto _exit; + // statistic + block.nRow += pBlockData->nRow; + for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { + TSDBKEY key = {.ts = pBlockData->aTSKEY[iRow], .version = pBlockData->aVersion[iRow]}; + if (iRow == 0) { + if (tsdbKeyCmprFn(&block.minKey, &key) > 0) { + block.minKey = key; + } + } else { + if (pBlockData->aTSKEY[iRow] == pBlockData->aTSKEY[iRow - 1]) { + block.hasDup = 1; + } + } + + if (iRow == pBlockData->nRow - 1 && tsdbKeyCmprFn(&block.maxKey, &key) < 0) { + block.maxKey = key; + } + + block.minVer = TMIN(block.minVer, key.version); + block.maxVer = TMAX(block.maxVer, key.version); + } + + // write + code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &block.aSubBlock[block.nSubBlock++], + &block.smaInfo, pCommitter->cmprAlg, 0, NULL); + if (code) goto _err; + + // put SBlock code = tMapDataPutItem(&pCommitter->dWriter.mBlock, &block, tPutBlock); - if (code) goto _exit; + if (code) goto _err; - tBlockDataClearData(&pCommitter->dWriter.bData); + // clear + tBlockDataClearData(pBlockData); -_exit: + return code; + +_err: + tsdbError("vgId:%d tsdb commit data block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); return code; } static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { - int32_t code = 0; - SBlockL blockL; + int32_t code = 0; + SBlockL blockL; + SBlockData *pBlockData = &pCommitter->dWriter.bDatal; - ASSERT(pCommitter->dWriter.bDatal.nRow > 0); + ASSERT(pBlockData->nRow > 0); - code = tsdbWriteLastBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, &blockL, NULL, NULL, - pCommitter->cmprAlg); - if (code) goto _exit; + // statistic + blockL.suid = pBlockData->suid; + blockL.nRow = pBlockData->nRow; + blockL.minVer = VERSION_MAX; + blockL.maxVer = VERSION_MIN; + for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { + blockL.minVer = TMIN(blockL.minVer, pBlockData->aVersion[iRow]); + blockL.maxVer = TMIN(blockL.maxVer, pBlockData->aVersion[iRow]); + } + blockL.minUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[0]; + blockL.maxUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[pBlockData->nRow - 1]; + // write + code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &blockL.bInfo, NULL, pCommitter->cmprAlg, 1, NULL); + if (code) goto _err; + + // push SBlockL if (taosArrayPush(pCommitter->dWriter.aBlockL, &blockL) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + goto _err; } - tBlockDataClearData(&pCommitter->dWriter.bDatal); + // clear + tBlockDataClearData(pBlockData); -_exit: + return code; + +_err: + tsdbError("vgId:%d tsdb commit last block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index eeb11abc97..c542ab5583 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -85,11 +85,11 @@ _err: } int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, uint8_t **ppBuf, SDelIdx *pDelIdx) { - int32_t code = 0; - uint8_t *pBuf = NULL; - int64_t size; - int64_t n; - SBlockDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pDelIdx->suid, .uid = pDelIdx->uid}; + int32_t code = 0; + uint8_t *pBuf = NULL; + int64_t size; + int64_t n; + SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pDelIdx->suid, .uid = pDelIdx->uid}; if (!ppBuf) ppBuf = &pBuf; @@ -106,7 +106,7 @@ int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, uint8_t **ppBuf // build n = 0; - *(SBlockDataHdr *)(*ppBuf) = hdr; + *(SDiskDataHdr *)(*ppBuf) = hdr; n += sizeof(hdr); for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { n += tPutDelData(*ppBuf + n, taosArrayGet(aDelData, iDelData)); @@ -306,13 +306,13 @@ _exit: } int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, uint8_t **ppBuf) { - int32_t code = 0; - int64_t offset = pDelIdx->offset; - int64_t size = pDelIdx->size; - int64_t n; - uint8_t *pBuf = NULL; - SBlockDataHdr *pHdr; - SDelData *pDelData = &(SDelData){0}; + int32_t code = 0; + int64_t offset = pDelIdx->offset; + int64_t size = pDelIdx->size; + int64_t n; + uint8_t *pBuf = NULL; + SDiskDataHdr *pHdr; + SDelData *pDelData = &(SDelData){0}; if (!ppBuf) ppBuf = &pBuf; @@ -344,7 +344,7 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData // // decode n = 0; - pHdr = (SBlockDataHdr *)(*ppBuf + n); + pHdr = (SDiskDataHdr *)(*ppBuf + n); ASSERT(pHdr->delimiter == TSDB_FILE_DLMT); ASSERT(pHdr->suid == pDelIdx->suid); ASSERT(pHdr->uid == pDelIdx->uid); @@ -669,13 +669,13 @@ _err: } int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBlock, uint8_t **ppBuf) { - int32_t code = 0; - int64_t offset = pBlockIdx->offset; - int64_t size = pBlockIdx->size; - uint8_t *pBuf = NULL; - int64_t n; - int64_t tn; - SBlockDataHdr hdr; + int32_t code = 0; + int64_t offset = pBlockIdx->offset; + int64_t size = pBlockIdx->size; + uint8_t *pBuf = NULL; + int64_t n; + int64_t tn; + SDiskDataHdr hdr; if (!ppBuf) ppBuf = &pBuf; @@ -706,7 +706,7 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBl } // decode - hdr = *(SBlockDataHdr *)(*ppBuf); + hdr = *(SDiskDataHdr *)(*ppBuf); ASSERT(hdr.delimiter == TSDB_FILE_DLMT); ASSERT(hdr.suid == pBlockIdx->suid); ASSERT(hdr.uid == pBlockIdx->uid); @@ -729,7 +729,7 @@ _err: return code; } -static int32_t tsdbReadBlockDataKey(SBlockData *pBlockData, SSubBlock *pSubBlock, uint8_t *pBuf, uint8_t **ppBuf) { +static int32_t tsdbReadBlockDataKey(SBlockData *pBlockData, SBlockInfo *pSubBlock, uint8_t *pBuf, uint8_t **ppBuf) { int32_t code = 0; int64_t size = pSubBlock->szVersion + pSubBlock->szTSKEY + sizeof(TSCKSUM); int64_t n; @@ -784,7 +784,7 @@ _err: return code; } -static int32_t tsdbReadColDataImpl(SSubBlock *pSubBlock, SBlockCol *pBlockCol, SColData *pColData, uint8_t *pBuf, +static int32_t tsdbReadColDataImpl(SBlockInfo *pSubBlock, SBlockCol *pBlockCol, SColData *pColData, uint8_t *pBuf, uint8_t **ppBuf) { int32_t code = 0; int64_t size; @@ -877,7 +877,7 @@ _err: return code; } -static int32_t tsdbReadBlockCol(uint8_t *pBuf, int32_t szBlockCol, SBlockDataHdr *pHdr, SArray *aBlockCol) { +static int32_t tsdbReadBlockCol(uint8_t *pBuf, int32_t szBlockCol, SDiskDataHdr *pHdr, SArray *aBlockCol) { int32_t code = 0; int32_t n = 0; SBlockCol blockCol; @@ -890,8 +890,8 @@ static int32_t tsdbReadBlockCol(uint8_t *pBuf, int32_t szBlockCol, SBlockDataHdr } // hdr - *pHdr = *(SBlockDataHdr *)pBuf; - n += sizeof(SBlockDataHdr); + *pHdr = *(SDiskDataHdr *)pBuf; + n += sizeof(SDiskDataHdr); // aBlockCol while (n < szBlockCol) { @@ -952,13 +952,13 @@ _exit: static int32_t tsdbReadSubColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int32_t iSubBlock, int16_t *aColId, int32_t nCol, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2) { - TdFilePtr pFD = pReader->pDataFD; - SSubBlock *pSubBlock = &pBlock->aSubBlock[iSubBlock]; - SArray *aBlockCol = NULL; - int32_t code = 0; - int64_t offset; - int64_t size; - int64_t n; + TdFilePtr pFD = pReader->pDataFD; + SBlockInfo *pSubBlock = &pBlock->aSubBlock[iSubBlock]; + SArray *aBlockCol = NULL; + int32_t code = 0; + int64_t offset; + int64_t size; + int64_t n; tBlockDataReset(pBlockData); pBlockData->nRow = pSubBlock->nRow; @@ -1123,13 +1123,13 @@ _err: static int32_t tsdbReadSubBlockData(SDataFReader *pReader, SBlock *pBlock, int32_t iSubBlock, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2) { - int32_t code = 0; - uint8_t *p; - int64_t size; - int64_t n; - TdFilePtr pFD = pReader->pDataFD; - SSubBlock *pSubBlock = &pBlock->aSubBlock[iSubBlock]; - SArray *aBlockCol = NULL; + int32_t code = 0; + uint8_t *p; + int64_t size; + int64_t n; + TdFilePtr pFD = pReader->pDataFD; + SBlockInfo *pSubBlock = &pBlock->aSubBlock[iSubBlock]; + SArray *aBlockCol = NULL; tBlockDataReset(pBlockData); @@ -1300,9 +1300,9 @@ int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *p } // decode block col - SBlockDataHdr hdr; - SArray *aBlockCol = taosArrayInit(0, sizeof(SBlockCol)); - uint8_t *p = *ppBuf1; + SDiskDataHdr hdr; + SArray *aBlockCol = taosArrayInit(0, sizeof(SBlockCol)); + uint8_t *p = *ppBuf1; if (aBlockCol == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -1461,6 +1461,8 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } + code = tDiskDataInit(&pWriter->dData); + if (code) goto _err; pWriter->pTsdb = pTsdb; pWriter->wSet = (SDFileSet){.diskId = pSet->diskId, .fid = pSet->fid, @@ -1639,6 +1641,7 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { goto _err; } + tDiskDataClear(&(*ppWriter)->dData); taosMemoryFree(*ppWriter); _exit: *ppWriter = NULL; @@ -1786,17 +1789,17 @@ _err: } int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, uint8_t **ppBuf, SBlockIdx *pBlockIdx) { - int32_t code = 0; - SHeadFile *pHeadFile = &pWriter->fHead; - SBlockDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pBlockIdx->suid, .uid = pBlockIdx->uid}; - uint8_t *pBuf = NULL; - int64_t size; - int64_t n; + int32_t code = 0; + SHeadFile *pHeadFile = &pWriter->fHead; + SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pBlockIdx->suid, .uid = pBlockIdx->uid}; + uint8_t *pBuf = NULL; + int64_t size; + int64_t n; ASSERT(mBlock->nItem > 0); // prepare - size = sizeof(SBlockDataHdr) + tPutMapData(NULL, mBlock) + sizeof(TSCKSUM); + size = sizeof(SDiskDataHdr) + tPutMapData(NULL, mBlock) + sizeof(TSCKSUM); // alloc if (!ppBuf) ppBuf = &pBuf; @@ -1805,7 +1808,7 @@ int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, uint8_t **ppBuf, // build n = 0; - *(SBlockDataHdr *)(*ppBuf) = hdr; + *(SDiskDataHdr *)(*ppBuf) = hdr; n += sizeof(hdr); n += tPutMapData(*ppBuf + n, mBlock); taosCalcChecksumAppend(0, *ppBuf, size); @@ -1915,141 +1918,7 @@ static void tsdbUpdateBlockInfo(SBlockData *pBlockData, SBlock *pBlock) { pBlock->nRow += pBlockData->nRow; } -static int32_t tsdbWriteDataArray(uint8_t *aData, int32_t nEle, int8_t type, int8_t cmprAlg, int32_t *rSize, - uint8_t **ppBuf1, int64_t nBuf1, uint8_t **ppBuf2) { - int32_t code = 0; - int32_t size; - - if (IS_VAR_DATA_TYPE(type)) { - size = nEle; - } else { - size = tDataTypes[type].bytes * nEle; - } - - if (cmprAlg == NO_COMPRESSION) { - code = tRealloc(ppBuf1, nBuf1 + size); - if (code) goto _exit; - - memcpy(*ppBuf1 + nBuf1, aData, size); - *rSize = size; - } else { - code = tRealloc(ppBuf1, size + COMP_OVERFLOW_BYTES); - if (code) goto _exit; - - if (cmprAlg == TWO_STAGE_COMP) { - code = tRealloc(ppBuf2, size + COMP_OVERFLOW_BYTES); - if (code) goto _exit; - } - - int32_t n = tDataTypes[type].compFunc(aData, tDataTypes[type].bytes * nEle, nEle, *ppBuf1 + nBuf1, - size + COMP_OVERFLOW_BYTES, cmprAlg, *ppBuf2, size + COMP_OVERFLOW_BYTES); - if (n <= 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _exit; - } - *rSize = n; - } - -_exit: - return code; -} - -static int32_t tsdbWriteColData(SColData *pColData, SBlockCol *pBlockCol, int8_t cmprAlg, uint8_t **ppBuf1, - int64_t nBuf1, uint8_t **ppBuf2) { - int32_t code = 0; - int64_t size; - int64_t n = 0; - - // BITMAP - if (pColData->flag != HAS_VALUE) { - code = tsdbWriteDataArray(pColData->pBitMap, BIT2_SIZE(pColData->nVal), TSDB_DATA_TYPE_TINYINT, cmprAlg, - &pBlockCol->szBitmap, ppBuf1, nBuf1 + n, ppBuf2); - if (code) goto _err; - } else { - pBlockCol->szBitmap = 0; - } - n += pBlockCol->szBitmap; - - // OFFSET - if (IS_VAR_DATA_TYPE(pColData->type)) { - code = tsdbWriteDataArray((uint8_t *)pColData->aOffset, pColData->nVal, TSDB_DATA_TYPE_INT, cmprAlg, - &pBlockCol->szOffset, ppBuf1, nBuf1 + n, ppBuf2); - if (code) goto _err; - } else { - pBlockCol->szOffset = 0; - } - n += pBlockCol->szOffset; - - // VALUE - if (pColData->flag != (HAS_NULL | HAS_NONE)) { - ASSERT(pColData->nData); - code = tsdbWriteDataArray(pColData->pData, pColData->nData, pColData->type, cmprAlg, &pBlockCol->szValue, ppBuf1, - nBuf1 + n, ppBuf2); - if (code) goto _err; - } else { - ASSERT(pColData->nData == 0); - pBlockCol->szValue = 0; - } - n += pBlockCol->szValue; - - // checksum - n += sizeof(TSCKSUM); - code = tRealloc(ppBuf1, nBuf1 + n); - if (code) goto _err; - taosCalcChecksumAppend(0, *ppBuf1 + nBuf1, n); - - return code; - -_err: - return code; -} - -static int32_t tsdbWriteBlockDataImpl(TdFilePtr pFD, SBlockDataHdr hdr, SArray *aBlockCol, uint8_t *pData, - int64_t nData, uint8_t **ppBuf, int32_t *szBlockCol) { - int32_t code = 0; - int32_t nBlockCol = taosArrayGetSize(aBlockCol); - int64_t size; - int64_t n; - - // HDR + SArray - *szBlockCol = sizeof(hdr); - for (int32_t iBlockCol = 0; iBlockCol < nBlockCol; iBlockCol++) { - (*szBlockCol) += tPutBlockCol(NULL, taosArrayGet(aBlockCol, iBlockCol)); - } - - code = tRealloc(ppBuf, *szBlockCol + sizeof(TSCKSUM)); - if (code) goto _err; - - n = 0; - memcpy(*ppBuf, &hdr, sizeof(hdr)); - n += sizeof(hdr); - for (int32_t iBlockCol = 0; iBlockCol < nBlockCol; iBlockCol++) { - n += tPutBlockCol(*ppBuf + n, taosArrayGet(aBlockCol, iBlockCol)); - } - taosCalcChecksumAppend(0, *ppBuf, *szBlockCol + sizeof(TSCKSUM)); - - ASSERT(n == *szBlockCol); - - n = taosWriteFile(pFD, *ppBuf, *szBlockCol + sizeof(TSCKSUM)); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // SBlockData - n = taosWriteFile(pFD, pData, nData); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - return code; - -_err: - return code; -} - -static int32_t tsdbWriteBlockSma(TdFilePtr pFD, SBlockData *pBlockData, SSubBlock *pSubBlock, uint8_t **ppBuf) { +static int32_t tsdbWriteBlockSma(TdFilePtr pFD, SBlockData *pBlockData, SBlockInfo *pSubBlock, uint8_t **ppBuf) { int32_t code = 0; int64_t n; SColData *pColData; @@ -2093,230 +1962,93 @@ _err: return code; } -int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock *pBlock, uint8_t **ppBuf1, - uint8_t **ppBuf2, int8_t cmprAlg) { - int32_t code = 0; +int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, + int8_t cmprAlg, int8_t toLast, uint8_t **ppBuf) { + int32_t code = 0; + TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; + SDiskData *pDiskData = &pWriter->dData; + uint8_t *pBuf = NULL; - ASSERT((pBlockData->suid && pBlockData->uid) || (!pBlockData->suid && pBlockData->uid)); + if (!ppBuf) ppBuf = &pBuf; - uint8_t *pBuf1 = NULL; - uint8_t *pBuf2 = NULL; - - if (!ppBuf1) ppBuf1 = &pBuf1; - if (!ppBuf2) ppBuf2 = &pBuf2; - - tsdbUpdateBlockInfo(pBlockData, pBlock); - - SSubBlock *pSubBlock = &pBlock->aSubBlock[pBlock->nSubBlock++]; - - pSubBlock->nRow = pBlockData->nRow; - pSubBlock->cmprAlg = cmprAlg; - pSubBlock->offset = pWriter->fData.size; - - // ======================= BLOCK DATA ======================= - int64_t nBuf1 = 0; - - // VERSION - code = tsdbWriteDataArray((uint8_t *)pBlockData->aVersion, pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, - &pSubBlock->szVersion, ppBuf1, nBuf1, ppBuf2); + // convert + code = tBlockToDiskData(pBlockData, pDiskData, cmprAlg); if (code) goto _err; - nBuf1 += pSubBlock->szVersion; - // TSKEY - code = tsdbWriteDataArray((uint8_t *)pBlockData->aTSKEY, pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg, - &pSubBlock->szTSKEY, ppBuf1, nBuf1, ppBuf2); + // write the block + if (toLast) { + pBlkInfo->offset = pWriter->fLast.size; + } else { + pBlkInfo->offset = pWriter->fData.size; + } + + // HDR and KEY + int32_t size = tPutDiskDataHdr(NULL, &pDiskData->hdr); + code = tRealloc(ppBuf, size); if (code) goto _err; - nBuf1 += pSubBlock->szTSKEY; - // checksum - nBuf1 += sizeof(TSCKSUM); - code = tRealloc(ppBuf1, nBuf1); - if (code) goto _err; - taosCalcChecksumAppend(0, *ppBuf1, nBuf1); + tPutDiskDataHdr(*ppBuf, &pDiskData->hdr); - // COLUMNS - SArray *aBlockCol = taosArrayInit(taosArrayGetSize(pBlockData->aIdx), sizeof(SBlockCol)); - if (aBlockCol == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + TSCKSUM cksm = taosCalcChecksum(0, *ppBuf, size); + + int64_t n = taosWriteFile(pFD, *ppBuf, size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); goto _err; } - int32_t offset = 0; - for (int32_t iCol = 0; iCol < taosArrayGetSize(pBlockData->aIdx); iCol++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iCol); - SBlockCol blockCol = {0}; - ASSERT(pColData->flag); - - if (pColData->flag == HAS_NONE) continue; - - blockCol.cid = pColData->cid; - blockCol.type = pColData->type; - blockCol.smaOn = pColData->smaOn; - blockCol.flag = pColData->flag; - blockCol.szOrigin = pColData->nData; - - if (pColData->flag != HAS_NULL) { - code = tsdbWriteColData(pColData, &blockCol, cmprAlg, ppBuf1, nBuf1, ppBuf2); - if (code) goto _err; - - blockCol.offset = offset; - offset = offset + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); - nBuf1 = nBuf1 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); - } - - if (taosArrayPush(aBlockCol, &blockCol) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } + cksm = taosCalcChecksum(cksm, *pDiskData->ppKey, pDiskData->hdr.szUid + pDiskData->hdr.szVer + pDiskData->hdr.szKey); + n = taosWriteFile(pFD, *pDiskData->ppKey, pDiskData->hdr.szUid + pDiskData->hdr.szVer + pDiskData->hdr.szKey); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; } - // write - SBlockDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pBlockData->suid, .uid = pBlockData->uid}; - code = tsdbWriteBlockDataImpl(pWriter->pDataFD, hdr, aBlockCol, *ppBuf1, nBuf1, ppBuf2, &pSubBlock->szBlockCol); - if (code) goto _err; - - pSubBlock->szBlock = pSubBlock->szBlockCol + sizeof(TSCKSUM) + nBuf1; - pWriter->fData.size += pSubBlock->szBlock; - - // ======================= BLOCK SMA ======================= - pSubBlock->sOffset = 0; - pSubBlock->nSma = 0; - - if (pBlock->nSubBlock > 1 || pBlock->hasDup) goto _exit; - - code = tsdbWriteBlockSma(pWriter->pSmaFD, pBlockData, pSubBlock, ppBuf1); - if (code) goto _err; - - if (pSubBlock->nSma > 0) { - pSubBlock->sOffset = pWriter->fSma.size; - pWriter->fSma.size += (sizeof(SColumnDataAgg) * pSubBlock->nSma + sizeof(TSCKSUM)); + n = taosWriteFile(pFD, &cksm, sizeof(cksm)); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; } + pBlkInfo->szKey = size + pDiskData->hdr.szUid + pDiskData->hdr.szVer + pDiskData->hdr.szKey + sizeof(TSCKSUM); + + // SBlockCol + if (pDiskData->hdr.szBlkCol == 0) { + pBlkInfo->szBlock = pBlkInfo->szKey; + goto _write_sma; + } + + code = tRealloc(ppBuf, pDiskData->hdr.szBlkCol + sizeof(TSCKSUM)); + if (code) goto _err; + + n = 0; + for (int32_t iBlockCol = 0; iBlockCol < taosArrayGetSize(pDiskData->aBlockCol); iBlockCol++) { + n += tPutBlockCol(*ppBuf + n, taosArrayGet(pDiskData->aBlockCol, iBlockCol)); + } + taosCalcChecksumAppend(0, *ppBuf, pDiskData->hdr.szBlkCol + sizeof(TSCKSUM)); + + n = taosWriteFile(pFD, *ppBuf, pDiskData->hdr.szBlkCol + sizeof(TSCKSUM)); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + for (int32_t iBlockCol = 0; iBlockCol < taosArrayGetSize(pDiskData->aBlockCol); iBlockCol++) { + } + +// ================= SMA ==================== +_write_sma: + if (toLast) goto _exit; + if (pSmaInfo == NULL) goto _exit; + + // TODO + _exit: - tFree(pBuf1); - tFree(pBuf2); - taosArrayDestroy(aBlockCol); + tFree(pBuf); return code; _err: - tsdbError("vgId:%d, write block data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf1); - tFree(pBuf2); - taosArrayDestroy(aBlockCol); - return code; -} - -int32_t tsdbWriteLastBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockL *pBlockL, uint8_t **ppBuf1, - uint8_t **ppBuf2, int8_t cmprAlg) { - int32_t code = 0; - - ASSERT((pBlockData->suid && !pBlockData->uid) || (!pBlockData->suid && pBlockData->uid)); - - uint8_t *pBuf1 = NULL; - uint8_t *pBuf2 = NULL; - - if (!ppBuf1) ppBuf1 = &pBuf1; - if (!ppBuf2) ppBuf2 = &pBuf2; - - pBlockL->suid = pBlockData->suid; - if (pBlockData->uid) { - pBlockL->maxUid = pBlockL->minUid = pBlockData->uid; - } else { - pBlockL->minUid = pBlockData->aUid[0]; - pBlockL->maxUid = pBlockData->aUid[pBlockData->nRow - 1]; - } - pBlockL->minVer = VERSION_MAX; - pBlockL->maxVer = VERSION_MIN; - pBlockL->nRow = pBlockData->nRow; - pBlockL->offset = pWriter->fLast.size; - pBlockL->cmprAlg = cmprAlg; - for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { - pBlockL->minVer = TMIN(pBlockL->minVer, pBlockData->aVersion[iRow]); - pBlockL->maxVer = TMAX(pBlockL->maxVer, pBlockData->aVersion[iRow]); - } - - // ======================= BLOCK DATA ======================= - int64_t nBuf1 = 0; - - // UID - if (pBlockData->uid == 0) { - code = tsdbWriteDataArray((uint8_t *)pBlockData->aUid, pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, - &pBlockL->szUid, ppBuf1, nBuf1, ppBuf2); - if (code) goto _err; - } else { - pBlockL->szUid = 0; - } - nBuf1 += pBlockL->szUid; - - // VERSION - code = tsdbWriteDataArray((uint8_t *)pBlockData->aVersion, pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, - &pBlockL->szVer, ppBuf1, nBuf1, ppBuf2); - if (code) goto _err; - nBuf1 += pBlockL->szVer; - - // TSKEY - code = tsdbWriteDataArray((uint8_t *)pBlockData->aTSKEY, pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg, - &pBlockL->szTSKEY, ppBuf1, nBuf1, ppBuf2); - if (code) goto _err; - nBuf1 += pBlockL->szTSKEY; - - // checksum - nBuf1 += sizeof(TSCKSUM); - code = tRealloc(ppBuf1, nBuf1); - if (code) goto _err; - taosCalcChecksumAppend(0, *ppBuf1, nBuf1); - - // COLUMNS - SArray *aBlockCol = taosArrayInit(taosArrayGetSize(pBlockData->aIdx), sizeof(SBlockCol)); - if (aBlockCol == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - int32_t offset = 0; - for (int32_t iCol = 0; iCol < taosArrayGetSize(pBlockData->aIdx); iCol++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iCol); - SBlockCol blockCol = {0}; - - ASSERT(pColData->flag); - - if (pColData->flag == HAS_NONE) continue; - - blockCol.cid = pColData->cid; - blockCol.type = pColData->type; - blockCol.smaOn = pColData->smaOn; - blockCol.flag = pColData->flag; - blockCol.szOrigin = pColData->nData; - - if (pColData->flag != HAS_NULL) { - code = tsdbWriteColData(pColData, &blockCol, cmprAlg, ppBuf1, nBuf1, ppBuf2); - if (code) goto _err; - - blockCol.offset = offset; - offset = offset + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); - nBuf1 = nBuf1 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); - } - - if (taosArrayPush(aBlockCol, &blockCol) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - } - - // write - SBlockDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pBlockData->suid, .uid = pBlockData->uid}; - code = tsdbWriteBlockDataImpl(pWriter->pLastFD, hdr, aBlockCol, *ppBuf1, nBuf1, ppBuf2, &pBlockL->szBlockCol); - if (code) goto _err; - - pBlockL->szBlock = pBlockL->szBlockCol + sizeof(TSCKSUM) + nBuf1; - pWriter->fLast.size += pBlockL->szBlock; - - tFree(pBuf1); - tFree(pBuf2); - return code; - -_err: - tFree(pBuf1); - tFree(pBuf2); + tFree(pBuf); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 13d293f86b..72d9175078 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -217,31 +217,30 @@ int32_t tCmprBlockIdx(void const *lhs, void const *rhs) { // SBlock ====================================================== void tBlockReset(SBlock *pBlock) { - *pBlock = - (SBlock){.minKey = TSDBKEY_MAX, .maxKey = TSDBKEY_MIN, .minVersion = VERSION_MAX, .maxVersion = VERSION_MIN}; + *pBlock = (SBlock){.minKey = TSDBKEY_MAX, .maxKey = TSDBKEY_MIN, .minVer = VERSION_MAX, .maxVer = VERSION_MIN}; } int32_t tPutBlock(uint8_t *p, void *ph) { int32_t n = 0; SBlock *pBlock = (SBlock *)ph; - n += tPutTSDBKEY(p ? p + n : p, &pBlock->minKey); - n += tPutTSDBKEY(p ? p + n : p, &pBlock->maxKey); + n += tPutI64v(p ? p + n : p, pBlock->minKey.version); + n += tPutI64(p ? p + n : p, pBlock->minKey.ts); + n += tPutI64v(p ? p + n : p, pBlock->maxKey.version); + n += tPutI64(p ? p + n : p, pBlock->maxKey.ts); n += tPutI64v(p ? p + n : p, pBlock->minVer); n += tPutI64v(p ? p + n : p, pBlock->maxVer); n += tPutI32v(p ? p + n : p, pBlock->nRow); n += tPutI8(p ? p + n : p, pBlock->hasDup); n += tPutI8(p ? p + n : p, pBlock->nSubBlock); for (int8_t iSubBlock = 0; iSubBlock < pBlock->nSubBlock; iSubBlock++) { - n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].nRow); - n += tPutI8(p ? p + n : p, pBlock->aSubBlock[iSubBlock].cmprAlg); n += tPutI64v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].offset); - n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].szBlockCol); - n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].szVersion); - n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].szTSKEY); n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].szBlock); - n += tPutI64v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].sOffset); - n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].nSma); + n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].szKey); + } + if (pBlock->nSubBlock == 1 && !pBlock->hasDup) { + n += tPutI64v(p ? p + n : p, pBlock->smaInfo.offset); + n += tPutI32v(p ? p + n : p, pBlock->smaInfo.size); } return n; @@ -251,23 +250,26 @@ int32_t tGetBlock(uint8_t *p, void *ph) { int32_t n = 0; SBlock *pBlock = (SBlock *)ph; - n += tGetTSDBKEY(p + n, &pBlock->minKey); - n += tGetTSDBKEY(p + n, &pBlock->maxKey); + n += tGetI64v(p + n, &pBlock->minKey.version); + n += tGetI64(p + n, &pBlock->minKey.ts); + n += tGetI64v(p + n, &pBlock->maxKey.version); + n += tGetI64(p + n, &pBlock->maxKey.ts); n += tGetI64v(p + n, &pBlock->minVer); n += tGetI64v(p + n, &pBlock->maxVer); n += tGetI32v(p + n, &pBlock->nRow); n += tGetI8(p + n, &pBlock->hasDup); n += tGetI8(p + n, &pBlock->nSubBlock); for (int8_t iSubBlock = 0; iSubBlock < pBlock->nSubBlock; iSubBlock++) { - n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].nRow); - n += tGetI8(p + n, &pBlock->aSubBlock[iSubBlock].cmprAlg); n += tGetI64v(p + n, &pBlock->aSubBlock[iSubBlock].offset); - n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].szBlockCol); - n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].szVersion); - n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].szTSKEY); n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].szBlock); - n += tGetI64v(p + n, &pBlock->aSubBlock[iSubBlock].sOffset); - n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].nSma); + n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].szKey); + } + if (pBlock->nSubBlock == 1 && !pBlock->hasDup) { + n += tGetI64v(p + n, &pBlock->smaInfo.offset); + n += tGetI32v(p + n, &pBlock->smaInfo.size); + } else { + pBlock->smaInfo.offset = 0; + pBlock->smaInfo.size = 0; } return n; @@ -290,8 +292,9 @@ bool tBlockHasSma(SBlock *pBlock) { if (pBlock->nSubBlock > 1) return false; if (pBlock->hasDup) return false; - return pBlock->aSubBlock[0].nSma > 0; + return pBlock->smaInfo.size > 0; } + // SBlockL ====================================================== int32_t tPutBlockL(uint8_t *p, void *ph) { int32_t n = 0; @@ -303,13 +306,9 @@ int32_t tPutBlockL(uint8_t *p, void *ph) { n += tPutI64v(p ? p + n : p, pBlockL->minVer); n += tPutI64v(p ? p + n : p, pBlockL->maxVer); n += tPutI32v(p ? p + n : p, pBlockL->nRow); - n += tPutI64v(p ? p + n : p, pBlockL->offset); - n += tPutI8(p ? p + n : p, pBlockL->cmprAlg); - n += tPutI32v(p ? p + n : p, pBlockL->szBlockCol); - n += tPutI32v(p ? p + n : p, pBlockL->szUid); - n += tPutI32v(p ? p + n : p, pBlockL->szVer); - n += tPutI32v(p ? p + n : p, pBlockL->szTSKEY); - n += tPutI32v(p ? p + n : p, pBlockL->szBlock); + n += tPutI64v(p ? p + n : p, pBlockL->bInfo.offset); + n += tPutI32v(p ? p + n : p, pBlockL->bInfo.szBlock); + n += tPutI32v(p ? p + n : p, pBlockL->bInfo.szKey); return n; } @@ -324,13 +323,9 @@ int32_t tGetBlockL(uint8_t *p, void *ph) { n += tGetI64v(p + n, &pBlockL->minVer); n += tGetI64v(p + n, &pBlockL->maxVer); n += tGetI32v(p + n, &pBlockL->nRow); - n += tGetI64v(p + n, &pBlockL->offset); - n += tGetI8(p + n, &pBlockL->cmprAlg); - n += tGetI32v(p + n, &pBlockL->szBlockCol); - n += tGetI32v(p + n, &pBlockL->szUid); - n += tGetI32v(p + n, &pBlockL->szVer); - n += tGetI32v(p + n, &pBlockL->szTSKEY); - n += tGetI32v(p + n, &pBlockL->szBlock); + n += tGetI64v(p + n, &pBlockL->bInfo.offset); + n += tGetI32v(p + n, &pBlockL->bInfo.szBlock); + n += tGetI32v(p + n, &pBlockL->bInfo.szKey); return n; } @@ -346,15 +341,25 @@ int32_t tPutBlockCol(uint8_t *p, void *ph) { n += tPutI8(p ? p + n : p, pBlockCol->type); n += tPutI8(p ? p + n : p, pBlockCol->smaOn); n += tPutI8(p ? p + n : p, pBlockCol->flag); + n += tPutI32v(p ? p + n : p, pBlockCol->szOrigin); if (pBlockCol->flag != HAS_NULL) { - n += tPutI32v(p ? p + n : p, pBlockCol->szOrigin); - n += tPutI32v(p ? p + n : p, pBlockCol->szBitmap); - n += tPutI32v(p ? p + n : p, pBlockCol->szOffset); - n += tPutI32v(p ? p + n : p, pBlockCol->szValue); + if (pBlockCol->flag != HAS_VALUE) { + n += tPutI32v(p ? p + n : p, pBlockCol->szBitmap); + } + + if (IS_VAR_DATA_TYPE(pBlockCol->type)) { + n += tPutI32v(p ? p + n : p, pBlockCol->szOffset); + } + + if (pBlockCol->flag != (HAS_NULL | HAS_NONE)) { + n += tPutI32v(p ? p + n : p, pBlockCol->szValue); + } + n += tPutI32v(p ? p + n : p, pBlockCol->offset); } +_exit: return n; } @@ -366,14 +371,28 @@ int32_t tGetBlockCol(uint8_t *p, void *ph) { n += tGetI8(p + n, &pBlockCol->type); n += tGetI8(p + n, &pBlockCol->smaOn); n += tGetI8(p + n, &pBlockCol->flag); + n += tGetI32v(p + n, &pBlockCol->szOrigin); ASSERT(pBlockCol->flag && (pBlockCol->flag != HAS_NONE)); + pBlockCol->szBitmap = 0; + pBlockCol->szOffset = 0; + pBlockCol->szValue = 0; + pBlockCol->offset = 0; + if (pBlockCol->flag != HAS_NULL) { - n += tGetI32v(p + n, &pBlockCol->szOrigin); - n += tGetI32v(p + n, &pBlockCol->szBitmap); - n += tGetI32v(p + n, &pBlockCol->szOffset); - n += tGetI32v(p + n, &pBlockCol->szValue); + if (pBlockCol->flag != HAS_VALUE) { + n += tGetI32v(p + n, &pBlockCol->szBitmap); + } + + if (IS_VAR_DATA_TYPE(pBlockCol->type)) { + n += tGetI32v(p + n, &pBlockCol->szOffset); + } + + if (pBlockCol->flag != (HAS_NULL | HAS_NONE)) { + n += tGetI32v(p + n, &pBlockCol->szValue); + } + n += tGetI32v(p + n, &pBlockCol->offset); } @@ -1650,23 +1669,36 @@ int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cm taosArrayClear(pDiskData->aBlockCol); pDiskData->nBuf = 0; - // uid - if (pDiskData->uid == 0) { - code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, - &pDiskData->pUid, &pDiskData->szUid, &pDiskData->pBuf); + { + pDiskData->ppKey = tDiskDataAllocBuf(pDiskData); + if (pDiskData->ppKey == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + int32_t n = 0; + // uid + if (pDiskData->uid == 0) { + code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, + cmprAlg, pDiskData->ppKey, n, &pDiskData->szUid, NULL); + if (code) goto _exit; + } else { + pDiskData->szUid = 0; + } + n += pDiskData->szUid; + + // version + code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, + cmprAlg, pDiskData->ppKey, n, &pDiskData->szVer, NULL); + if (code) goto _exit; + n += pDiskData->szVer; + + // tskey + code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, + cmprAlg, pDiskData->ppKey, &pDiskData->szKey, NULL); if (code) goto _exit; } - // version - code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, - cmprAlg, &pDiskData->pVer, &pDiskData->szVer, &pDiskData->pBuf); - if (code) goto _exit; - - // tskey - code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, - cmprAlg, &pDiskData->pKey, &pDiskData->szKey, &pDiskData->pBuf); - if (code) goto _exit; - // columns int32_t offset = 0; for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { @@ -1689,7 +1721,7 @@ int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cm } // compress - code = tsdbCmprColData(pColData, cmprAlg, &blockCol); + code = tsdbCmprColData(pColData, cmprAlg, &blockCol, NULL); if (code) goto _exit; // update offset @@ -1709,6 +1741,41 @@ _exit: int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cmprAlg); int32_t tDiskToBlockData(SDiskData *pDiskData, SBlockData *pBlockData); +// SDiskDataHdr ============================== +int32_t tPutDiskDataHdr(uint8_t *p, void *ph) { + int32_t n = 0; + SDiskDataHdr *pHdr = (SDiskDataHdr *)ph; + + n += tPutU32(p ? p + n : p, pHdr->delimiter); + n += tPutI64(p ? p + n : p, pHdr->suid); + n += tPutI64(p ? p + n : p, pHdr->uid); + n += tPutI32v(p ? p + n : p, pHdr->szUid); + n += tPutI32v(p ? p + n : p, pHdr->szVer); + n += tPutI32v(p ? p + n : p, pHdr->szKey); + n += tPutI32v(p ? p + n : p, pHdr->szBlkCol); + n += tPutI32v(p ? p + n : p, pHdr->nRow); + n += tPutI8(p ? p + n : p, pHdr->cmprAlg); + + return n; +} + +int32_t tGetDiskDataHdr(uint8_t *p, void *ph) { + int32_t n = 0; + SDiskDataHdr *pHdr = (SDiskDataHdr *)ph; + + n += tGetU32(p + n, &pHdr->delimiter); + n += tGetI64(p + n, &pHdr->suid); + n += tGetI64(p + n, &pHdr->uid); + n += tGetI32v(p + n, &pHdr->szUid); + n += tGetI32v(p + n, &pHdr->szVer); + n += tGetI32v(p + n, &pHdr->szKey); + n += tGetI32v(p + n, &pHdr->szBlkCol); + n += tGetI32v(p + n, &pHdr->nRow); + n += tGetI8(p + n, &pHdr->cmprAlg); + + return n; +} + // ALGORITHM ============================== void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { SColVal colVal; From 85907bc2696c173345ee76e5332613c0963f8944 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 6 Aug 2022 02:03:20 +0000 Subject: [PATCH 14/92] more work --- source/dnode/vnode/src/inc/tsdb.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 08331b1327..bb08a345c2 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -62,6 +62,8 @@ typedef struct SRowIter SRowIter; typedef struct STsdbFS STsdbFS; typedef struct SRowMerger SRowMerger; typedef struct STsdbReadSnap STsdbReadSnap; +typedef struct SBlockInfo SBlockInfo; +typedef struct SSmaInfo SSmaInfo; #define TSDB_MAX_SUBBLOCKS 8 #define TSDB_FHDR_SIZE 512 @@ -411,16 +413,16 @@ typedef struct { uint8_t **ppData; } SBlockCol; -typedef struct { +struct SBlockInfo { int64_t offset; // block data offset int32_t szBlock; int32_t szKey; -} SBlockInfo; +}; -typedef struct { +struct SSmaInfo { int64_t offset; int32_t size; -} SSmaInfo; +}; struct SBlock { TSDBKEY minKey; From 04386a3003fee0c41c95313a62730dea22f3e456 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 6 Aug 2022 07:45:24 +0000 Subject: [PATCH 15/92] more work --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index cd94762c7b..aa76ab1e27 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -530,8 +530,10 @@ static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) { } // write - code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &block.aSubBlock[block.nSubBlock++], - &block.smaInfo, pCommitter->cmprAlg, 0, NULL); + block.nSubBlock++; + code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &block.aSubBlock[block.nSubBlock - 1], + ((block.nSubBlock == 1) && block.hasDup) ? &block.smaInfo : NULL, pCommitter->cmprAlg, 0, + NULL); if (code) goto _err; // put SBlock @@ -562,7 +564,7 @@ static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { blockL.maxVer = VERSION_MIN; for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { blockL.minVer = TMIN(blockL.minVer, pBlockData->aVersion[iRow]); - blockL.maxVer = TMIN(blockL.maxVer, pBlockData->aVersion[iRow]); + blockL.maxVer = TMAX(blockL.maxVer, pBlockData->aVersion[iRow]); } blockL.minUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[0]; blockL.maxUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[pBlockData->nRow - 1]; From 3324f4e5b7a979323984c348a512263ef8537e0a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 6 Aug 2022 12:04:42 +0000 Subject: [PATCH 16/92] more work --- source/dnode/vnode/src/inc/tsdb.h | 17 +- source/dnode/vnode/src/tsdb/tsdbCache.c | 6 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 23 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 226 ++++++++---------- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 12 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 51 ++-- 7 files changed, 162 insertions(+), 177 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index bb08a345c2..bd24b45ccd 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -183,6 +183,8 @@ void tsdbFidKeyRange(int32_t fid, int32_t minutes, int8_t precision, TSKEY *m int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t now); int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SArray *aSkyline); void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg); +int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); +int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); // tsdbMemTable ============================================================================================== // SMemTable int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); @@ -238,9 +240,9 @@ int32_t tsdbFSUpsertDelFile(STsdbFS *pFS, SDelFile *pDelFile); int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync); int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter); -int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **ppBuf); -int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, uint8_t **ppBuf, SBlockIdx *pBlockIdx); -int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL, uint8_t **ppBuf); +int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx); +int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, SBlockIdx *pBlockIdx); +int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL); int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, int8_t cmprAlg, int8_t toLast, uint8_t **ppBuf); @@ -248,9 +250,9 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); // SDataFReader int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFReaderClose(SDataFReader **ppReader); -int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppBuf); -int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf); -int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData, uint8_t **ppBuf); +int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx); +int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL); +int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData); int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int16_t *aColId, int32_t nCol, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, SBlockData *pBlockData, @@ -605,6 +607,9 @@ struct SDataFWriter { SLastFile fLast; SSmaFile fSma; + uint8_t *pBuf1; + uint8_t *pBuf2; + SDiskData dData; }; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index f03b02af27..1fc5eae13a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -465,13 +465,12 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { if (code) goto _err; // tMapDataReset(&state->blockIdxMap); - // code = tsdbReadBlockIdx(state->pDataFReader, &state->blockIdxMap, NULL); if (!state->aBlockIdx) { state->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); } else { taosArrayClear(state->aBlockIdx); } - code = tsdbReadBlockIdx(state->pDataFReader, state->aBlockIdx, NULL); + code = tsdbReadBlockIdx(state->pDataFReader, state->aBlockIdx); if (code) goto _err; /* if (state->pBlockIdx) { */ @@ -487,8 +486,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { } tMapDataReset(&state->blockMap); - code = tsdbReadBlock(state->pDataFReader, state->pBlockIdx, &state->blockMap, NULL); - /* code = tsdbReadBlock(state->pDataFReader, &state->blockIdx, &state->blockMap, NULL); */ + code = tsdbReadBlock(state->pDataFReader, state->pBlockIdx, &state->blockMap); if (code) goto _err; state->nBlock = state->blockMap.nItem; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index aa76ab1e27..5470e43400 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -339,7 +339,7 @@ static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) { pCommitter->dReader.pBlockIdx = (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx); - code = tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock, NULL); + code = tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); if (code) goto _exit; ASSERT(pCommitter->dReader.mBlock.nItem > 0); @@ -370,7 +370,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { if (code) goto _err; // data - code = tsdbReadBlockIdx(pCommitter->dReader.pReader, pCommitter->dReader.aBlockIdx, NULL); + code = tsdbReadBlockIdx(pCommitter->dReader.pReader, pCommitter->dReader.aBlockIdx); if (code) goto _err; pCommitter->dReader.iBlockIdx = 0; @@ -378,8 +378,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { pCommitter->dReader.pBlockIdx = (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx); - code = - tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock, NULL); + code = tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); if (code) goto _err; } else { pCommitter->dReader.pBlockIdx = NULL; @@ -387,7 +386,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { tBlockDataReset(&pCommitter->dReader.bData); // last - code = tsdbReadBlockL(pCommitter->dReader.pReader, pCommitter->dReader.aBlockL, NULL); + code = tsdbReadBlockL(pCommitter->dReader.pReader, pCommitter->dReader.aBlockL); if (code) goto _err; pCommitter->dReader.iBlockL = -1; @@ -506,7 +505,7 @@ static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) { tBlockReset(&block); // as a new block } - // statistic + // info block.nRow += pBlockData->nRow; for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { TSDBKEY key = {.ts = pBlockData->aTSKEY[iRow], .version = pBlockData->aVersion[iRow]}; @@ -532,7 +531,7 @@ static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) { // write block.nSubBlock++; code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &block.aSubBlock[block.nSubBlock - 1], - ((block.nSubBlock == 1) && block.hasDup) ? &block.smaInfo : NULL, pCommitter->cmprAlg, 0, + ((block.nSubBlock == 1) && !block.hasDup) ? &block.smaInfo : NULL, pCommitter->cmprAlg, 0, NULL); if (code) goto _err; @@ -557,7 +556,7 @@ static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { ASSERT(pBlockData->nRow > 0); - // statistic + // info blockL.suid = pBlockData->suid; blockL.nRow = pBlockData->nRow; blockL.minVer = VERSION_MAX; @@ -1041,7 +1040,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { // end if (pCommitter->dWriter.mBlock.nItem > 0) { SBlockIdx blockIdx = {.suid = pTbData->suid, .uid = pTbData->uid}; - code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, NULL, &blockIdx); + code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx); if (code) goto _err; if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { @@ -1067,11 +1066,11 @@ static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { int32_t code = 0; // write aBlockIdx - code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx, NULL); + code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx); if (code) goto _err; // write aBlockL - code = tsdbWriteBlockL(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockL, NULL); + code = tsdbWriteBlockL(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockL); if (code) goto _err; // update file header @@ -1107,7 +1106,7 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { if (pCommitter->dReader.pBlockIdx == NULL || tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) >= 0) break; SBlockIdx blockIdx = *pCommitter->dReader.pBlockIdx; - code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, NULL, &blockIdx); + code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx); if (code) goto _err; if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 84ecaa605d..df61d3d4c9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -545,7 +545,7 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader, SArray* aBlockIdx = taosArrayInit(8, sizeof(SBlockIdx)); int64_t st = taosGetTimestampUs(); - int32_t code = tsdbReadBlockIdx(pFileReader, aBlockIdx, NULL); + int32_t code = tsdbReadBlockIdx(pFileReader, aBlockIdx); if (code != TSDB_CODE_SUCCESS) { goto _end; } @@ -617,7 +617,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, uint32_ STableBlockScanInfo* pScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(int64_t)); tMapDataReset(&pScanInfo->mapData); - tsdbReadBlock(pReader->pFileReader, pBlockIdx, &pScanInfo->mapData, NULL); + tsdbReadBlock(pReader->pFileReader, pBlockIdx, &pScanInfo->mapData); size += pScanInfo->mapData.nData; for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index c542ab5583..4db0333890 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -441,6 +441,9 @@ struct SDataFReader { TdFilePtr pDataFD; TdFilePtr pLastFD; TdFilePtr pSmaFD; + + uint8_t *pBuf1; + uint8_t *pBuf2; }; int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) { @@ -523,6 +526,8 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { goto _err; } + tFree((*ppReader)->pBuf1); + tFree((*ppReader)->pBuf2); taosMemoryFree(*ppReader); _exit: @@ -534,14 +539,12 @@ _err: return code; } -int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppBuf) { - int32_t code = 0; - int64_t offset = pReader->pSet->pHeadF->offset; - int64_t size = pReader->pSet->pHeadF->loffset - offset; - uint8_t *pBuf = NULL; - int64_t n; - uint32_t delimiter; - SBlockIdx blockIdx; +int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { + int32_t code = 0; + int64_t offset = pReader->pSet->pHeadF->offset; + int64_t size = pReader->pSet->pHeadF->loffset - offset; + int64_t n; + uint32_t delimiter; taosArrayClear(aBlockIdx); if (size == 0) { @@ -549,8 +552,7 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppB } // alloc - if (!ppBuf) ppBuf = &pBuf; - code = tRealloc(ppBuf, size); + code = tRealloc(&pReader->pBuf1, size); if (code) goto _err; // seek @@ -560,7 +562,7 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppB } // read - n = taosReadFile(pReader->pHeadFD, *ppBuf, size); + n = taosReadFile(pReader->pHeadFD, pReader->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -570,18 +572,19 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppB } // check - if (!taosCheckChecksumWhole(*ppBuf, size)) { + if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } // decode n = 0; - n = tGetU32(*ppBuf + n, &delimiter); + n = tGetU32(pReader->pBuf1 + n, &delimiter); ASSERT(delimiter == TSDB_FILE_DLMT); while (n < size - sizeof(TSCKSUM)) { - n += tGetBlockIdx(*ppBuf + n, &blockIdx); + SBlockIdx blockIdx; + n += tGetBlockIdx(pReader->pBuf1 + n, &blockIdx); if (taosArrayPush(aBlockIdx, &blockIdx) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -591,24 +594,20 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppB ASSERT(n + sizeof(TSCKSUM) == size); - tFree(pBuf); _exit: return code; _err: tsdbError("vgId:%d, read block idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf); return code; } -int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf) { +int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL) { int32_t code = 0; int64_t offset = pReader->pSet->pHeadF->loffset; int64_t size = pReader->pSet->pHeadF->size - offset; int64_t n; uint32_t delimiter; - uint8_t *pBuf = NULL; - SBlockL blockl; taosArrayClear(aBlockL); if (size == 0) { @@ -616,8 +615,7 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf) } // alloc - if (!ppBuf) ppBuf = &pBuf; - code = tRealloc(ppBuf, size); + code = tRealloc(&pReader->pBuf1, size); if (code) goto _err; // seek @@ -627,7 +625,7 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf) } // read - n = taosReadFile(pReader->pHeadFD, *ppBuf, size); + n = taosReadFile(pReader->pHeadFD, pReader->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -637,18 +635,19 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf) } // check - if (!taosCheckChecksumWhole(*ppBuf, size)) { + if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } // decode n = 0; - n = tGetU32(*ppBuf + n, &delimiter); + n = tGetU32(pReader->pBuf1 + n, &delimiter); ASSERT(delimiter == TSDB_FILE_DLMT); while (n < size - sizeof(TSCKSUM)) { - n += tGetBlockL(*ppBuf + n, &blockl); + SBlockL blockl; + n += tGetBlockL(pReader->pBuf1 + n, &blockl); if (taosArrayPush(aBlockL, &blockl) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -658,29 +657,23 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf) ASSERT(n + sizeof(TSCKSUM) == size); - tFree(pBuf); _exit: return code; _err: tsdbError("vgId:%d read blockl failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf); return code; } -int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBlock, uint8_t **ppBuf) { - int32_t code = 0; - int64_t offset = pBlockIdx->offset; - int64_t size = pBlockIdx->size; - uint8_t *pBuf = NULL; - int64_t n; - int64_t tn; - SDiskDataHdr hdr; - - if (!ppBuf) ppBuf = &pBuf; +int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBlock) { + int32_t code = 0; + int64_t offset = pBlockIdx->offset; + int64_t size = pBlockIdx->size; + int64_t n; + int64_t tn; // alloc - code = tRealloc(ppBuf, size); + code = tRealloc(&pReader->pBuf1, size); if (code) goto _err; // seek @@ -690,7 +683,7 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBl } // read - n = taosReadFile(pReader->pHeadFD, *ppBuf, size); + n = taosReadFile(pReader->pHeadFD, pReader->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -700,19 +693,19 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBl } // check - if (!taosCheckChecksumWhole(*ppBuf, size)) { + if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } // decode - hdr = *(SDiskDataHdr *)(*ppBuf); - ASSERT(hdr.delimiter == TSDB_FILE_DLMT); - ASSERT(hdr.suid == pBlockIdx->suid); - ASSERT(hdr.uid == pBlockIdx->uid); + n = 0; - n = sizeof(hdr); - tn = tGetMapData(*ppBuf + n, mBlock); + uint32_t delimiter; + n += tGetU32(pReader->pBuf1 + n, &delimiter); + ASSERT(delimiter == TSDB_FILE_DLMT); + + tn = tGetMapData(pReader->pBuf1 + n, mBlock); if (tn < 0) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -720,12 +713,10 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBl n += tn; ASSERT(n + sizeof(TSCKSUM) == size); - tFree(pBuf); return code; _err: tsdbError("vgId:%d, read block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf); return code; } @@ -1642,6 +1633,8 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { } tDiskDataClear(&(*ppWriter)->dData); + tFree((*ppWriter)->pBuf1); + tFree((*ppWriter)->pBuf2); taosMemoryFree(*ppWriter); _exit: *ppWriter = NULL; @@ -1732,10 +1725,9 @@ _err: return code; } -int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **ppBuf) { +int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) { int32_t code = 0; SHeadFile *pHeadFile = &pWriter->fHead; - uint8_t *pBuf = NULL; int64_t size; int64_t n; @@ -1746,29 +1738,28 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **pp } // prepare - size = tPutU32(NULL, TSDB_FILE_DLMT); + size = sizeof(uint32_t); for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) { size += tPutBlockIdx(NULL, taosArrayGet(aBlockIdx, iBlockIdx)); } size += sizeof(TSCKSUM); // alloc - if (!ppBuf) ppBuf = &pBuf; - code = tRealloc(ppBuf, size); + code = tRealloc(&pWriter->pBuf1, size); if (code) goto _err; // build n = 0; - n = tPutU32(*ppBuf + n, TSDB_FILE_DLMT); + n = tPutU32(pWriter->pBuf1 + n, TSDB_FILE_DLMT); for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) { - n += tPutBlockIdx(*ppBuf + n, taosArrayGet(aBlockIdx, iBlockIdx)); + n += tPutBlockIdx(pWriter->pBuf1 + n, taosArrayGet(aBlockIdx, iBlockIdx)); } - taosCalcChecksumAppend(0, *ppBuf, size); + taosCalcChecksumAppend(0, pWriter->pBuf1, size); ASSERT(n + sizeof(TSCKSUM) == size); // write - n = taosWriteFile(pWriter->pHeadFD, *ppBuf, size); + n = taosWriteFile(pWriter->pHeadFD, pWriter->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1779,44 +1770,36 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **pp pHeadFile->size += size; _exit: - tFree(pBuf); return code; _err: tsdbError("vgId:%d, write block idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf); return code; } -int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, uint8_t **ppBuf, SBlockIdx *pBlockIdx) { - int32_t code = 0; - SHeadFile *pHeadFile = &pWriter->fHead; - SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pBlockIdx->suid, .uid = pBlockIdx->uid}; - uint8_t *pBuf = NULL; - int64_t size; - int64_t n; +int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, SBlockIdx *pBlockIdx) { + int32_t code = 0; + SHeadFile *pHeadFile = &pWriter->fHead; + int64_t size; + int64_t n; ASSERT(mBlock->nItem > 0); - // prepare - size = sizeof(SDiskDataHdr) + tPutMapData(NULL, mBlock) + sizeof(TSCKSUM); - // alloc - if (!ppBuf) ppBuf = &pBuf; - code = tRealloc(ppBuf, size); + size = sizeof(uint32_t) + tPutMapData(NULL, mBlock) + sizeof(TSCKSUM); + code = tRealloc(&pWriter->pBuf1, size); if (code) goto _err; // build n = 0; - *(SDiskDataHdr *)(*ppBuf) = hdr; - n += sizeof(hdr); - n += tPutMapData(*ppBuf + n, mBlock); - taosCalcChecksumAppend(0, *ppBuf, size); + n += tPutU32(pWriter->pBuf1 + n, TSDB_FILE_DLMT); + n += tPutMapData(pWriter->pBuf1 + n, mBlock); + taosCalcChecksumAppend(0, pWriter->pBuf1, size); ASSERT(n + sizeof(TSCKSUM) == size); // write - n = taosWriteFile(pWriter->pHeadFD, *ppBuf, size); + n = taosWriteFile(pWriter->pHeadFD, pWriter->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1827,21 +1810,18 @@ int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, uint8_t **ppBuf, pBlockIdx->size = size; pHeadFile->size += size; - tFree(pBuf); tsdbTrace("vgId:%d, write block, offset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), pBlockIdx->offset, pBlockIdx->size); return code; _err: - tFree(pBuf); tsdbError("vgId:%d, write block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); return code; } -int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL, uint8_t **ppBuf) { +int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) { int32_t code = 0; SHeadFile *pHeadFile = &pWriter->fHead; - uint8_t *pBuf = NULL; int64_t size; int64_t n; @@ -1852,29 +1832,28 @@ int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL, uint8_t **ppBuf) } // size - size = sizeof(uint32_t); + size = sizeof(uint32_t); // TSDB_FILE_DLMT for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aBlockL); iBlockL++) { size += tPutBlockL(NULL, taosArrayGet(aBlockL, iBlockL)); } size += sizeof(TSCKSUM); // alloc - if (!ppBuf) ppBuf = &pBuf; - code = tRealloc(ppBuf, size); + code = tRealloc(&pWriter->pBuf1, size); if (code) goto _err; // encode n = 0; - n += tPutU32(*ppBuf + n, TSDB_FILE_DLMT); + n += tPutU32(pWriter->pBuf1 + n, TSDB_FILE_DLMT); for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aBlockL); iBlockL++) { - n += tPutBlockL(*ppBuf + n, taosArrayGet(aBlockL, iBlockL)); + n += tPutBlockL(pWriter->pBuf1 + n, taosArrayGet(aBlockL, iBlockL)); } - taosCalcChecksumAppend(0, *ppBuf, size); + taosCalcChecksumAppend(0, pWriter->pBuf1, size); ASSERT(n + sizeof(TSCKSUM) == size); // write - n = taosWriteFile(pWriter->pHeadFD, *ppBuf, size); + n = taosWriteFile(pWriter->pHeadFD, pWriter->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1885,12 +1864,10 @@ int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL, uint8_t **ppBuf) pHeadFile->size += size; _exit: - tFree(pBuf); return code; _err: tsdbError("vgId:%d tsdb write blockl failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf); return code; } @@ -1918,47 +1895,49 @@ static void tsdbUpdateBlockInfo(SBlockData *pBlockData, SBlock *pBlock) { pBlock->nRow += pBlockData->nRow; } -static int32_t tsdbWriteBlockSma(TdFilePtr pFD, SBlockData *pBlockData, SBlockInfo *pSubBlock, uint8_t **ppBuf) { - int32_t code = 0; - int64_t n; - SColData *pColData; +static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, SSmaInfo *pSmaInfo) { + int32_t code = 0; - // prepare - pSubBlock->nSma = 0; + pSmaInfo->offset = 0; + pSmaInfo->size = 0; + + // encode for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { - pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); + SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - if (IS_VAR_DATA_TYPE(pColData->type) || (!pColData->smaOn)) continue; + if ((!pColData->smaOn) || IS_VAR_DATA_TYPE(pColData->type)) continue; - pSubBlock->nSma++; + SColumnDataAgg sma; + tsdbCalcColDataSMA(pColData, &sma); + + code = tRealloc(&pWriter->pBuf1, pSmaInfo->size + tPutColumnDataAgg(NULL, &sma)); + if (code) goto _err; + pSmaInfo->size += tPutColumnDataAgg(pWriter->pBuf1 + pSmaInfo->size, &sma); } - if (pSubBlock->nSma == 0) goto _exit; - - // calc - code = tRealloc(ppBuf, sizeof(SColumnDataAgg) * pSubBlock->nSma + sizeof(TSCKSUM)); - if (code) goto _err; - n = 0; - for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { - pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - - if (IS_VAR_DATA_TYPE(pColData->type) || (!pColData->smaOn)) continue; - - tsdbCalcColDataSMA(pColData, &((SColumnDataAgg *)(*ppBuf))[n]); - n++; - } - taosCalcChecksumAppend(0, *ppBuf, sizeof(SColumnDataAgg) * pSubBlock->nSma + sizeof(TSCKSUM)); // write - n = taosWriteFile(pFD, *ppBuf, sizeof(SColumnDataAgg) * pSubBlock->nSma + sizeof(TSCKSUM)); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; + if (pSmaInfo->size) { + int32_t size = pSmaInfo->size + sizeof(TSCKSUM); + + code = tRealloc(&pWriter->pBuf1, size); + if (code) goto _err; + + taosCalcChecksumAppend(0, pWriter->pBuf1, size); + + int64_t n = taosWriteFile(pWriter->pSmaFD, pWriter->pBuf1, size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + pSmaInfo->offset = pWriter->fSma.size; + pWriter->fSma.size += size; } -_exit: return code; _err: + tsdbError("vgId:%d tsdb write block sma failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); return code; } @@ -2036,12 +2015,11 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock for (int32_t iBlockCol = 0; iBlockCol < taosArrayGetSize(pDiskData->aBlockCol); iBlockCol++) { } -// ================= SMA ==================== -_write_sma: - if (toLast) goto _exit; - if (pSmaInfo == NULL) goto _exit; - - // TODO + // ================= SMA ==================== + if (pSmaInfo) { + code = tsdbWriteBlockSma(pWriter, pBlockData, pSmaInfo); + if (code) goto _err; + } _exit: tFree(pBuf); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index eec7a3e4a6..2b5ea030b2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -58,7 +58,7 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { if (code) goto _err; // SBlockIdx - code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx, NULL); + code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx); if (code) goto _err; pReader->iBlockIdx = 0; @@ -78,7 +78,7 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx); pReader->iBlockIdx++; - code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock, NULL); + code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock); if (code) goto _err; pReader->iBlock = 0; @@ -552,7 +552,7 @@ _err: static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* pBlockIdx) { int32_t code = 0; - code = tsdbReadBlock(pWriter->pDataFReader, pBlockIdx, &pWriter->mBlock, NULL); + code = tsdbReadBlock(pWriter->pDataFReader, pBlockIdx, &pWriter->mBlock); if (code) goto _err; // SBlockData @@ -791,7 +791,7 @@ static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, TABLEID id) { } if (pWriter->pBlockIdx) { - code = tsdbReadBlock(pWriter->pDataFReader, pWriter->pBlockIdx, &pWriter->mBlock, NULL); + code = tsdbReadBlock(pWriter->pDataFReader, pWriter->pBlockIdx, &pWriter->mBlock); if (code) goto _err; } else { tMapDataReset(&pWriter->mBlock); @@ -843,7 +843,7 @@ static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) { pWriter->iBlockIdx++; } - code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdxW, NULL); + code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdxW); if (code) goto _err; code = tsdbFSUpsertFSet(&pWriter->fs, &pWriter->pDataFWriter->wSet); @@ -897,7 +897,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 code = tsdbDataFReaderOpen(&pWriter->pDataFReader, pTsdb, pSet); if (code) goto _err; - code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx, NULL); + code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx); if (code) goto _err; } else { ASSERT(pWriter->pDataFReader == NULL); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 72d9175078..0ec922f20e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -152,25 +152,6 @@ int32_t tTABLEIDCmprFn(const void *p1, const void *p2) { return 0; } -// TSDBKEY ====================================================== -static FORCE_INLINE int32_t tPutTSDBKEY(uint8_t *p, TSDBKEY *pKey) { - int32_t n = 0; - - n += tPutI64v(p ? p + n : p, pKey->version); - n += tPutI64(p ? p + n : p, pKey->ts); - - return n; -} - -static FORCE_INLINE int32_t tGetTSDBKEY(uint8_t *p, TSDBKEY *pKey) { - int32_t n = 0; - - n += tGetI64v(p + n, &pKey->version); - n += tGetI64(p + n, &pKey->ts); - - return n; -} - // SBlockIdx ====================================================== int32_t tPutBlockIdx(uint8_t *p, void *ph) { int32_t n = 0; @@ -225,9 +206,9 @@ int32_t tPutBlock(uint8_t *p, void *ph) { SBlock *pBlock = (SBlock *)ph; n += tPutI64v(p ? p + n : p, pBlock->minKey.version); - n += tPutI64(p ? p + n : p, pBlock->minKey.ts); + n += tPutI64v(p ? p + n : p, pBlock->minKey.ts); n += tPutI64v(p ? p + n : p, pBlock->maxKey.version); - n += tPutI64(p ? p + n : p, pBlock->maxKey.ts); + n += tPutI64v(p ? p + n : p, pBlock->maxKey.ts); n += tPutI64v(p ? p + n : p, pBlock->minVer); n += tPutI64v(p ? p + n : p, pBlock->maxVer); n += tPutI32v(p ? p + n : p, pBlock->nRow); @@ -251,9 +232,9 @@ int32_t tGetBlock(uint8_t *p, void *ph) { SBlock *pBlock = (SBlock *)ph; n += tGetI64v(p + n, &pBlock->minKey.version); - n += tGetI64(p + n, &pBlock->minKey.ts); + n += tGetI64v(p + n, &pBlock->minKey.ts); n += tGetI64v(p + n, &pBlock->maxKey.version); - n += tGetI64(p + n, &pBlock->maxKey.ts); + n += tGetI64v(p + n, &pBlock->maxKey.ts); n += tGetI64v(p + n, &pBlock->minVer); n += tGetI64v(p + n, &pBlock->maxVer); n += tGetI32v(p + n, &pBlock->nRow); @@ -1777,6 +1758,30 @@ int32_t tGetDiskDataHdr(uint8_t *p, void *ph) { } // ALGORITHM ============================== +int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) { + int32_t n = 0; + + n += tPutI16v(p ? p + n : p, pColAgg->colId); + n += tPutI16v(p ? p + n : p, pColAgg->numOfNull); + n += tPutI64(p ? p + n : p, pColAgg->sum); + n += tPutI64(p ? p + n : p, pColAgg->max); + n += tPutI64(p ? p + n : p, pColAgg->min); + + return n; +} + +int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) { + int32_t n = 0; + + n += tGetI16v(p + n, &pColAgg->colId); + n += tGetI16v(p + n, &pColAgg->numOfNull); + n += tGetI64(p + n, &pColAgg->sum); + n += tGetI64(p + n, &pColAgg->max); + n += tGetI64(p + n, &pColAgg->min); + + return n; +} + void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { SColVal colVal; SColVal *pColVal = &colVal; From 3436d52316d3882db8b0dfd3b96ca41ae5f7def7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 6 Aug 2022 12:23:29 +0000 Subject: [PATCH 17/92] more refact --- source/dnode/vnode/src/inc/tsdb.h | 12 +- source/dnode/vnode/src/tsdb/tsdbCache.c | 5 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 10 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 153 +++++++----------- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 14 +- 6 files changed, 78 insertions(+), 120 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index bd24b45ccd..60727b5e8e 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -265,14 +265,14 @@ int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *p // SDelFWriter int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync); -int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, uint8_t **ppBuf, SDelIdx *pDelIdx); -int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx, uint8_t **ppBuf); +int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelIdx); +int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx); int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter); // SDelFReader -int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb, uint8_t **ppBuf); +int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFReaderClose(SDelFReader **ppReader); -int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, uint8_t **ppBuf); -int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx, uint8_t **ppBuf); +int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData); +int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx); // tsdbRead.c ============================================================================================== int32_t tsdbTakeReadSnap(STsdb *pTsdb, STsdbReadSnap **ppSnap); void tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap); @@ -583,6 +583,8 @@ struct SDelFWriter { STsdb *pTsdb; SDelFile fDel; TdFilePtr pWriteH; + + uint8_t *pBuf1; }; struct SDiskData { diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 1fc5eae13a..5ad1f3bcb9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -317,7 +317,7 @@ static int32_t getTableDelDataFromDelIdx(SDelFReader *pDelReader, SDelIdx *pDelI int32_t code = 0; if (pDelIdx) { - code = tsdbReadDelData(pDelReader, pDelIdx, aDelData, NULL); + code = tsdbReadDelData(pDelReader, pDelIdx, aDelData); } return code; @@ -388,8 +388,7 @@ static int32_t getTableDelIdx(SDelFReader *pDelFReader, tb_uid_t suid, tb_uid_t SDelIdx idx = {.suid = suid, .uid = uid}; // tMapDataReset(&delIdxMap); - // code = tsdbReadDelIdx(pDelFReader, &delIdxMap, NULL); - code = tsdbReadDelIdx(pDelFReader, pDelIdxArray, NULL); + code = tsdbReadDelIdx(pDelFReader, pDelIdxArray); if (code) goto _err; // code = tMapDataSearch(&delIdxMap, &idx, tGetDelIdx, tCmprDelIdx, pDelIdx); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 5470e43400..3bb78af44a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -181,10 +181,10 @@ static int32_t tsdbCommitDelStart(SCommitter *pCommitter) { SDelFile *pDelFileR = pCommitter->fs.pDelFile; if (pDelFileR) { - code = tsdbDelFReaderOpen(&pCommitter->pDelFReader, pDelFileR, pTsdb, NULL); + code = tsdbDelFReaderOpen(&pCommitter->pDelFReader, pDelFileR, pTsdb); if (code) goto _err; - code = tsdbReadDelIdx(pCommitter->pDelFReader, pCommitter->aDelIdx, NULL); + code = tsdbReadDelIdx(pCommitter->pDelFReader, pCommitter->aDelIdx); if (code) goto _err; } @@ -221,7 +221,7 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel suid = pDelIdx->suid; uid = pDelIdx->uid; - code = tsdbReadDelData(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData, NULL); + code = tsdbReadDelData(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData); if (code) goto _err; } else { taosArrayClear(pCommitter->aDelData); @@ -241,7 +241,7 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel } // write - code = tsdbWriteDelData(pCommitter->pDelFWriter, pCommitter->aDelData, NULL, &delIdx); + code = tsdbWriteDelData(pCommitter->pDelFWriter, pCommitter->aDelData, &delIdx); if (code) goto _err; // put delIdx @@ -262,7 +262,7 @@ static int32_t tsdbCommitDelEnd(SCommitter *pCommitter) { int32_t code = 0; STsdb *pTsdb = pCommitter->pTsdb; - code = tsdbWriteDelIdx(pCommitter->pDelFWriter, pCommitter->aDelIdxN, NULL); + code = tsdbWriteDelIdx(pCommitter->pDelFWriter, pCommitter->aDelIdxN); if (code) goto _err; code = tsdbUpdateDelFileHdr(pCommitter->pDelFWriter); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index df61d3d4c9..4523dda2b3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1618,7 +1618,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* goto _err; } - code = tsdbReadDelIdx(pDelFReader, aDelIdx, NULL); + code = tsdbReadDelIdx(pDelFReader, aDelIdx); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(aDelIdx); tsdbDelFReaderClose(&pDelFReader); @@ -1629,7 +1629,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* SDelIdx* pIdx = taosArraySearch(aDelIdx, &idx, tCmprDelIdx, TD_EQ); if (pIdx != NULL) { - code = tsdbReadDelData(pDelFReader, pIdx, pDelData, NULL); + code = tsdbReadDelData(pDelFReader, pIdx, pDelData); } taosArrayDestroy(aDelIdx); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 4db0333890..8ee8b085c0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -76,6 +76,9 @@ int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync) { goto _err; } + tFree(pWriter->pBuf1); + taosMemoryFree(pWriter); + *ppWriter = NULL; return code; @@ -84,39 +87,34 @@ _err: return code; } -int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, uint8_t **ppBuf, SDelIdx *pDelIdx) { - int32_t code = 0; - uint8_t *pBuf = NULL; - int64_t size; - int64_t n; - SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pDelIdx->suid, .uid = pDelIdx->uid}; - - if (!ppBuf) ppBuf = &pBuf; +int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelIdx) { + int32_t code = 0; + int64_t size; + int64_t n; // prepare - size = sizeof(hdr); + size = sizeof(uint32_t); for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { size += tPutDelData(NULL, taosArrayGet(aDelData, iDelData)); } size += sizeof(TSCKSUM); // alloc - code = tRealloc(ppBuf, size); + code = tRealloc(&pWriter->pBuf1, size); if (code) goto _err; // build n = 0; - *(SDiskDataHdr *)(*ppBuf) = hdr; - n += sizeof(hdr); + n += tPutU32(pWriter->pBuf1 + n, TSDB_FILE_DLMT); for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { - n += tPutDelData(*ppBuf + n, taosArrayGet(aDelData, iDelData)); + n += tPutDelData(pWriter->pBuf1 + n, taosArrayGet(aDelData, iDelData)); } - taosCalcChecksumAppend(0, *ppBuf, size); + taosCalcChecksumAppend(0, pWriter->pBuf1, size); ASSERT(n + sizeof(TSCKSUM) == size); // write - n = taosWriteFile(pWriter->pWriteH, *ppBuf, size); + n = taosWriteFile(pWriter->pWriteH, pWriter->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -129,48 +127,42 @@ int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, uint8_t **ppBuf pDelIdx->size = size; pWriter->fDel.size += size; - tFree(pBuf); return code; _err: tsdbError("vgId:%d, failed to write del data since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf); return code; } -int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx, uint8_t **ppBuf) { +int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx) { int32_t code = 0; int64_t size; int64_t n; - uint8_t *pBuf = NULL; SDelIdx *pDelIdx; - if (!ppBuf) ppBuf = &pBuf; - // prepare - size = 0; - size += tPutU32(NULL, TSDB_FILE_DLMT); + size = sizeof(uint32_t); for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { size += tPutDelIdx(NULL, taosArrayGet(aDelIdx, iDelIdx)); } size += sizeof(TSCKSUM); // alloc - code = tRealloc(ppBuf, size); + code = tRealloc(&pWriter->pBuf1, size); if (code) goto _err; // build n = 0; - n += tPutU32(*ppBuf + n, TSDB_FILE_DLMT); + n += tPutU32(pWriter->pBuf1 + n, TSDB_FILE_DLMT); for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { - n += tPutDelIdx(*ppBuf + n, taosArrayGet(aDelIdx, iDelIdx)); + n += tPutDelIdx(pWriter->pBuf1 + n, taosArrayGet(aDelIdx, iDelIdx)); } - taosCalcChecksumAppend(0, *ppBuf, size); + taosCalcChecksumAppend(0, pWriter->pBuf1, size); ASSERT(n + sizeof(TSCKSUM) == size); // write - n = taosWriteFile(pWriter->pWriteH, *ppBuf, size); + n = taosWriteFile(pWriter->pWriteH, pWriter->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -180,12 +172,10 @@ int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx, uint8_t **ppBuf) pWriter->fDel.offset = pWriter->fDel.size; pWriter->fDel.size += size; - tFree(pBuf); return code; _err: tsdbError("vgId:%d, write del idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf); return code; } @@ -225,9 +215,11 @@ struct SDelFReader { STsdb *pTsdb; SDelFile fDel; TdFilePtr pReadH; + + uint8_t *pBuf1; }; -int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb, uint8_t **ppBuf) { +int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb) { int32_t code = 0; char fname[TSDB_FILENAME_LEN]; SDelFReader *pDelFReader; @@ -252,32 +244,6 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb goto _err; } -#if 0 - // load and check hdr if buffer is given - if (ppBuf) { - code = tRealloc(ppBuf, TSDB_FHDR_SIZE); - if (code) { - goto _err; - } - - n = taosReadFile(pDelFReader->pReadH, *ppBuf, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < TSDB_FHDR_SIZE) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - if (!taosCheckChecksumWhole(*ppBuf, TSDB_FHDR_SIZE)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // TODO: check the content - } -#endif - _exit: *ppReader = pDelFReader; return code; @@ -297,6 +263,7 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader) { code = TAOS_SYSTEM_ERROR(errno); goto _exit; } + tFree(pReader->pBuf1); taosMemoryFree(pReader); } *ppReader = NULL; @@ -305,16 +272,13 @@ _exit: return code; } -int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, uint8_t **ppBuf) { - int32_t code = 0; - int64_t offset = pDelIdx->offset; - int64_t size = pDelIdx->size; - int64_t n; - uint8_t *pBuf = NULL; - SDiskDataHdr *pHdr; - SDelData *pDelData = &(SDelData){0}; +int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData) { + int32_t code = 0; + int64_t offset = pDelIdx->offset; + int64_t size = pDelIdx->size; + int64_t n; - if (!ppBuf) ppBuf = &pBuf; + taosArrayClear(aDelData); // seek if (taosLSeekFile(pReader->pReadH, offset, SEEK_SET) < 0) { @@ -323,11 +287,11 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData } // alloc - code = tRealloc(ppBuf, size); + code = tRealloc(&pReader->pBuf1, size); if (code) goto _err; // read - n = taosReadFile(pReader->pReadH, *ppBuf, size); + n = taosReadFile(pReader->pReadH, pReader->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -337,23 +301,21 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData } // check - if (!taosCheckChecksumWhole(*ppBuf, size)) { + if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } // // decode n = 0; - pHdr = (SDiskDataHdr *)(*ppBuf + n); - ASSERT(pHdr->delimiter == TSDB_FILE_DLMT); - ASSERT(pHdr->suid == pDelIdx->suid); - ASSERT(pHdr->uid == pDelIdx->uid); - n += sizeof(*pHdr); - taosArrayClear(aDelData); - while (n < size - sizeof(TSCKSUM)) { - n += tGetDelData(*ppBuf + n, pDelData); - if (taosArrayPush(aDelData, pDelData) == NULL) { + uint32_t delimiter; + n += tGetU32(pReader->pBuf1 + n, &delimiter); + while (n < size - sizeof(TSCKSUM)) { + SDelData delData; + n += tGetDelData(pReader->pBuf1 + n, &delData); + + if (taosArrayPush(aDelData, &delData) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -361,25 +323,20 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData ASSERT(n == size - sizeof(TSCKSUM)); - tFree(pBuf); return code; _err: tsdbError("vgId:%d, read del data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf); return code; } -int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx, uint8_t **ppBuf) { - int32_t code = 0; - int32_t n; - int64_t offset = pReader->fDel.offset; - int64_t size = pReader->fDel.size - offset; - uint32_t delimiter; - uint8_t *pBuf = NULL; - SDelIdx *pDelIdx = &(SDelIdx){0}; +int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { + int32_t code = 0; + int32_t n; + int64_t offset = pReader->fDel.offset; + int64_t size = pReader->fDel.size - offset; - if (!ppBuf) ppBuf = &pBuf; + taosArrayClear(aDelIdx); // seek if (taosLSeekFile(pReader->pReadH, offset, SEEK_SET) < 0) { @@ -388,11 +345,11 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx, uint8_t **ppBuf) { } // alloc - code = tRealloc(ppBuf, size); + code = tRealloc(&pReader->pBuf1, size); if (code) goto _err; // read - n = taosReadFile(pReader->pReadH, *ppBuf, size); + n = taosReadFile(pReader->pReadH, pReader->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -402,21 +359,23 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx, uint8_t **ppBuf) { } // check - if (!taosCheckChecksumWhole(*ppBuf, size)) { + if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } // decode n = 0; - n += tGetU32(*ppBuf + n, &delimiter); + uint32_t delimiter; + n += tGetU32(pReader->pBuf1 + n, &delimiter); ASSERT(delimiter == TSDB_FILE_DLMT); - taosArrayClear(aDelIdx); while (n < size - sizeof(TSCKSUM)) { - n += tGetDelIdx(*ppBuf + n, pDelIdx); + SDelIdx delIdx; - if (taosArrayPush(aDelIdx, pDelIdx) == NULL) { + n += tGetDelIdx(pReader->pBuf1 + n, &delIdx); + + if (taosArrayPush(aDelIdx, &delIdx) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -424,12 +383,10 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx, uint8_t **ppBuf) { ASSERT(n == size - sizeof(TSCKSUM)); - tFree(pBuf); return code; _err: tsdbError("vgId:%d, read del idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 2b5ea030b2..06a38303ec 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -177,7 +177,7 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { if (code) goto _err; // read index - code = tsdbReadDelIdx(pReader->pDelFReader, pReader->aDelIdx, NULL); + code = tsdbReadDelIdx(pReader->pDelFReader, pReader->aDelIdx); if (code) goto _err; pReader->iDelIdx = 0; @@ -193,7 +193,7 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { pReader->iDelIdx++; - code = tsdbReadDelData(pReader->pDelFReader, pDelIdx, pReader->aDelData, NULL); + code = tsdbReadDelData(pReader->pDelFReader, pDelIdx, pReader->aDelData); if (code) goto _err; int32_t size = 0; @@ -968,7 +968,7 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 code = tsdbDelFReaderOpen(&pWriter->pDelFReader, pDelFile, pTsdb, NULL); if (code) goto _err; - code = tsdbReadDelIdx(pWriter->pDelFReader, pWriter->aDelIdxR, NULL); + code = tsdbReadDelIdx(pWriter->pDelFReader, pWriter->aDelIdxR); if (code) goto _err; } @@ -997,7 +997,7 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 if (c < 0) { goto _new_del; } else { - code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData, NULL); + code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData); if (code) goto _err; pWriter->iDelIdx++; @@ -1027,7 +1027,7 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 } _write_del: - code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, NULL, &delIdx); + code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx); if (code) goto _err; if (taosArrayPush(pWriter->aDelIdxW, &delIdx) == NULL) { @@ -1058,11 +1058,11 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) { for (; pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR); pWriter->iDelIdx++) { SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx); - code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData, NULL); + code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData); if (code) goto _err; SDelIdx delIdx = (SDelIdx){.suid = pDelIdx->suid, .uid = pDelIdx->uid}; - code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, NULL, &delIdx); + code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx); if (code) goto _err; if (taosArrayPush(pWriter->aDelIdxR, &delIdx) == NULL) { From 04804e384aa0ea2162848db100c31cd20aef6b56 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 6 Aug 2022 12:24:46 +0000 Subject: [PATCH 18/92] more --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 4523dda2b3..9c55ad2f54 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1607,7 +1607,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* SDelFile* pDelFile = pReader->pReadSnap->fs.pDelFile; if (pDelFile) { SDelFReader* pDelFReader = NULL; - code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb, NULL); + code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb); if (code != TSDB_CODE_SUCCESS) { goto _err; } From 43dec55fd740bfa1f5e067f585ccf2455d102ff1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 6 Aug 2022 12:48:58 +0000 Subject: [PATCH 19/92] more work --- source/dnode/vnode/src/inc/tsdb.h | 5 ++-- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 5 ++-- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 27 ++++++++++++------- 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 60727b5e8e..e7a6f5b8f2 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -244,15 +244,16 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx); int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, SBlockIdx *pBlockIdx); int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL); int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, - int8_t cmprAlg, int8_t toLast, uint8_t **ppBuf); + int8_t cmprAlg, int8_t toLast); int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); // SDataFReader int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFReaderClose(SDataFReader **ppReader); int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx); -int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL); int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData); +int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL); + int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int16_t *aColId, int32_t nCol, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, SBlockData *pBlockData, diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 5ad1f3bcb9..707aa3153d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -760,7 +760,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs if (pDelFile) { SDelFReader *pDelFReader; - code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb, NULL); + code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb); if (code) goto _err; code = getTableDelIdx(pDelFReader, suid, uid, &delIdx); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 3bb78af44a..4269a785a0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -531,8 +531,7 @@ static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) { // write block.nSubBlock++; code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &block.aSubBlock[block.nSubBlock - 1], - ((block.nSubBlock == 1) && !block.hasDup) ? &block.smaInfo : NULL, pCommitter->cmprAlg, 0, - NULL); + ((block.nSubBlock == 1) && !block.hasDup) ? &block.smaInfo : NULL, pCommitter->cmprAlg, 0); if (code) goto _err; // put SBlock @@ -569,7 +568,7 @@ static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { blockL.maxUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[pBlockData->nRow - 1]; // write - code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &blockL.bInfo, NULL, pCommitter->cmprAlg, 1, NULL); + code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &blockL.bInfo, NULL, pCommitter->cmprAlg, 1); if (code) goto _err; // push SBlockL diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 8ee8b085c0..8528765e74 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -679,6 +679,7 @@ _err: static int32_t tsdbReadBlockDataKey(SBlockData *pBlockData, SBlockInfo *pSubBlock, uint8_t *pBuf, uint8_t **ppBuf) { int32_t code = 0; +#if 0 int64_t size = pSubBlock->szVersion + pSubBlock->szTSKEY + sizeof(TSCKSUM); int64_t n; @@ -729,12 +730,14 @@ static int32_t tsdbReadBlockDataKey(SBlockData *pBlockData, SBlockInfo *pSubBloc return code; _err: +#endif return code; } static int32_t tsdbReadColDataImpl(SBlockInfo *pSubBlock, SBlockCol *pBlockCol, SColData *pColData, uint8_t *pBuf, uint8_t **ppBuf) { int32_t code = 0; +#if 0 int64_t size; int64_t n; @@ -822,6 +825,7 @@ static int32_t tsdbReadColDataImpl(SBlockInfo *pSubBlock, SBlockCol *pBlockCol, return code; _err: +#endif return code; } @@ -900,10 +904,11 @@ _exit: static int32_t tsdbReadSubColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int32_t iSubBlock, int16_t *aColId, int32_t nCol, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2) { + int32_t code = 0; +#if 0 TdFilePtr pFD = pReader->pDataFD; SBlockInfo *pSubBlock = &pBlock->aSubBlock[iSubBlock]; SArray *aBlockCol = NULL; - int32_t code = 0; int64_t offset; int64_t size; int64_t n; @@ -1012,6 +1017,7 @@ _exit: _err: taosArrayDestroy(aBlockCol); +#endif return code; } @@ -1071,7 +1077,8 @@ _err: static int32_t tsdbReadSubBlockData(SDataFReader *pReader, SBlock *pBlock, int32_t iSubBlock, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2) { - int32_t code = 0; + int32_t code = 0; +#if 0 uint8_t *p; int64_t size; int64_t n; @@ -1148,6 +1155,7 @@ static int32_t tsdbReadSubBlockData(SDataFReader *pReader, SBlock *pBlock, int32 _err: tsdbError("vgId:%d, tsdb read sub block data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); taosArrayDestroy(aBlockCol); +#endif return code; } @@ -1218,6 +1226,7 @@ _err: int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2) { int32_t code = 0; +#if 0 tBlockDataReset(pBlockData); @@ -1336,11 +1345,13 @@ int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *p _err: tsdbError("vgId:%d tsdb read last block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); +#endif return code; } int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg, uint8_t **ppBuf) { - int32_t code = 0; + int32_t code = 0; +#if 0 TdFilePtr pFD = pReader->pSmaFD; int64_t offset = pBlock->aSubBlock[0].sOffset; int64_t size = pBlock->aSubBlock[0].nSma * sizeof(SColumnDataAgg) + sizeof(TSCKSUM); @@ -1391,6 +1402,7 @@ int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnD _err: tsdbError("vgId:%d, read block sma failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); tFree(pBuf); +#endif return code; } @@ -1899,14 +1911,12 @@ _err: } int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, - int8_t cmprAlg, int8_t toLast, uint8_t **ppBuf) { + int8_t cmprAlg, int8_t toLast) { int32_t code = 0; TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; SDiskData *pDiskData = &pWriter->dData; - uint8_t *pBuf = NULL; - - if (!ppBuf) ppBuf = &pBuf; +#if 0 // convert code = tBlockToDiskData(pBlockData, pDiskData, cmprAlg); if (code) goto _err; @@ -1971,6 +1981,7 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock for (int32_t iBlockCol = 0; iBlockCol < taosArrayGetSize(pDiskData->aBlockCol); iBlockCol++) { } +#endif // ================= SMA ==================== if (pSmaInfo) { @@ -1979,11 +1990,9 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock } _exit: - tFree(pBuf); return code; _err: - tFree(pBuf); return code; } From 7c32e099d09b64d9cb689167ecec39ce9e7a493f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 6 Aug 2022 14:20:30 +0000 Subject: [PATCH 20/92] more work --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 56 +++++++++---------- 3 files changed, 28 insertions(+), 32 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index e7a6f5b8f2..2f440842c1 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -253,12 +253,12 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader); int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx); int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData); int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL); +int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg); int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int16_t *aColId, int32_t nCol, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); -int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg, uint8_t **ppBuf); int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, uint8_t **ppBuf1, diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 9c55ad2f54..87cc000908 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2806,7 +2806,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS SBlockLoadSuppInfo* pSup = &pReader->suppInfo; if (tBlockHasSma(pBlock)) { - code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pSup->pColAgg, NULL); + code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pSup->pColAgg); if (code != TSDB_CODE_SUCCESS) { tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64 ", code:%s, %s", 0, pFBlock->uid, tstrerror(code), pReader->idStr); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 8528765e74..8933aec100 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -829,6 +829,7 @@ _err: return code; } +#if 0 static int32_t tsdbReadBlockCol(uint8_t *pBuf, int32_t szBlockCol, SDiskDataHdr *pHdr, SArray *aBlockCol) { int32_t code = 0; int32_t n = 0; @@ -862,6 +863,7 @@ static int32_t tsdbReadBlockCol(uint8_t *pBuf, int32_t szBlockCol, SDiskDataHdr _err: return code; } +#endif static int32_t tsdbReadDataArray(uint8_t *pInput, int32_t szInput, int32_t nEle, int8_t type, int8_t cmprAlg, uint8_t **ppOut, uint8_t **ppBuf) { @@ -1349,30 +1351,21 @@ _err: return code; } -int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg, uint8_t **ppBuf) { - int32_t code = 0; -#if 0 - TdFilePtr pFD = pReader->pSmaFD; - int64_t offset = pBlock->aSubBlock[0].sOffset; - int64_t size = pBlock->aSubBlock[0].nSma * sizeof(SColumnDataAgg) + sizeof(TSCKSUM); - uint8_t *pBuf = NULL; - int64_t n; +int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg) { + int32_t code = 0; + SSmaInfo *pSmaInfo = &pBlock->smaInfo; - ASSERT(tBlockHasSma(pBlock)); + ASSERT(pSmaInfo->size > 0); - if (!ppBuf) ppBuf = &pBuf; - code = tRealloc(ppBuf, size); + taosArrayClear(aColumnDataAgg); + + // alloc + int32_t size = pSmaInfo->size + sizeof(TSCKSUM); + code = tRealloc(&pReader->pBuf1, size); if (code) goto _err; - // lseek - n = taosLSeekFile(pFD, offset, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - // read - n = taosReadFile(pFD, *ppBuf, size); + int64_t n = taosReadFile(pReader->pSmaFD, pReader->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1382,27 +1375,27 @@ int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnD } // check - if (!taosCheckChecksumWhole(*ppBuf, size)) { + if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } // decode - taosArrayClear(aColumnDataAgg); - for (int32_t iSma = 0; iSma < pBlock->aSubBlock[0].nSma; iSma++) { - if (taosArrayPush(aColumnDataAgg, &((SColumnDataAgg *)(*ppBuf))[iSma]) == NULL) { + n = 0; + while (n < pSmaInfo->size) { + SColumnDataAgg sma; + + n += tGetColumnDataAgg(pReader->pBuf1 + n, &sma); + if (taosArrayPush(aColumnDataAgg, &sma) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } } - tFree(pBuf); return code; _err: - tsdbError("vgId:%d, read block sma failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf); -#endif + tsdbError("vgId:%d tsdb read block sma failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; } @@ -1912,10 +1905,12 @@ _err: int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, int8_t cmprAlg, int8_t toLast) { - int32_t code = 0; - TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; - SDiskData *pDiskData = &pWriter->dData; + int32_t code = 0; + TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; + ASSERT(pBlockData->nRow > 0); + + // ================= DATA ==================== #if 0 // convert code = tBlockToDiskData(pBlockData, pDiskData, cmprAlg); @@ -1993,6 +1988,7 @@ _exit: return code; _err: + tsdbError("vgId:%d tsdb write block data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); return code; } From 85556a3c904ef864b4ecbb9dd084c79415612547 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 7 Aug 2022 06:13:51 +0000 Subject: [PATCH 21/92] more work --- source/dnode/vnode/src/inc/tsdb.h | 38 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 102 ++--- source/dnode/vnode/src/tsdb/tsdbUtil.c | 378 ++++++++---------- 3 files changed, 221 insertions(+), 297 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 2f440842c1..fe3992102b 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -47,7 +47,6 @@ typedef struct SBlockL SBlockL; typedef struct SColData SColData; typedef struct SDiskDataHdr SDiskDataHdr; typedef struct SBlockData SBlockData; -typedef struct SDiskData SDiskData; typedef struct SDelFile SDelFile; typedef struct SHeadFile SHeadFile; typedef struct SDataFile SDataFile; @@ -152,11 +151,6 @@ SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx); void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData); int32_t tPutBlockData(uint8_t *p, SBlockData *pBlockData); int32_t tGetBlockData(uint8_t *p, SBlockData *pBlockData); -// SDiskData -int32_t tDiskDataInit(SDiskData *pDiskData); -void tDiskDataClear(SDiskData *pDiskData); -int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cmprAlg); -int32_t tDiskToBlockData(SDiskData *pDiskData, SBlockData *pBlockData); // SDiskDataHdr int32_t tPutDiskDataHdr(uint8_t *p, void *ph); int32_t tGetDiskDataHdr(uint8_t *p, void *ph); @@ -185,6 +179,9 @@ int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SAr void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg); int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); +int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t nOut, + int32_t *szOut, uint8_t **ppBuf); +int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppBuf); // tsdbMemTable ============================================================================================== // SMemTable int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); @@ -404,16 +401,15 @@ struct SMapData { }; typedef struct { - int16_t cid; - int8_t type; - int8_t smaOn; - int8_t flag; // HAS_NONE|HAS_NULL|HAS_VALUE - int32_t szOrigin; // original column value size (only save for variant data type) - int32_t szBitmap; // bitmap size, 0 only for flag == HAS_VAL - int32_t szOffset; // offset size, 0 only for non-variant-length type - int32_t szValue; // value size, 0 when flag == (HAS_NULL | HAS_NONE) - int32_t offset; - uint8_t **ppData; + int16_t cid; + int8_t type; + int8_t smaOn; + int8_t flag; // HAS_NONE|HAS_NULL|HAS_VALUE + int32_t szOrigin; // original column value size (only save for variant data type) + int32_t szBitmap; // bitmap size, 0 only for flag == HAS_VAL + int32_t szOffset; // offset size, 0 only for non-variant-length type + int32_t szValue; // value size, 0 when flag == (HAS_NULL | HAS_NONE) + int32_t offset; } SBlockCol; struct SBlockInfo { @@ -588,14 +584,6 @@ struct SDelFWriter { uint8_t *pBuf1; }; -struct SDiskData { - SDiskDataHdr hdr; - uint8_t **ppKey; - SArray *aBlockCol; // SArray - int32_t nBuf; - SArray *aBuf; // SArray -}; - struct SDataFWriter { STsdb *pTsdb; SDFileSet wSet; @@ -612,8 +600,6 @@ struct SDataFWriter { uint8_t *pBuf1; uint8_t *pBuf2; - - SDiskData dData; }; struct STsdbReadSnap { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 8933aec100..31029e6bb4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1911,72 +1911,58 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock ASSERT(pBlockData->nRow > 0); // ================= DATA ==================== -#if 0 - // convert - code = tBlockToDiskData(pBlockData, pDiskData, cmprAlg); + SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, + .suid = pBlockData->suid, + .uid = pBlockData->uid, + .nRow = pBlockData->nRow, + .cmprAlg = cmprAlg}; + + SArray *aBlockCol = taosArrayInit(taosArrayGetSize(pBlockData->aIdx), sizeof(SBlockCol)); + if (aBlockCol == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + // uid + if (pBlockData->uid == 0) { + ASSERT(toLast); + code = tsdbCmprData(); + if (code) goto _err; + } + + // version + code = tsdbCmprData(); if (code) goto _err; - // write the block - if (toLast) { - pBlkInfo->offset = pWriter->fLast.size; - } else { - pBlkInfo->offset = pWriter->fData.size; - } - - // HDR and KEY - int32_t size = tPutDiskDataHdr(NULL, &pDiskData->hdr); - code = tRealloc(ppBuf, size); + // ts + code = tsdbCmprData(); if (code) goto _err; - tPutDiskDataHdr(*ppBuf, &pDiskData->hdr); + // columns + int32_t offset = 0; + for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { + SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - TSCKSUM cksm = taosCalcChecksum(0, *ppBuf, size); + ASSERT(pColData->flag); - int64_t n = taosWriteFile(pFD, *ppBuf, size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; + if (pColData->flag == HAS_NONE) continue; + + SBlockCol blockCol = {.cid = pColData->cid, + .type = pColData->type, + .smaOn = pColData->smaOn, + .flag = pColData->flag, + .szOrigin = pColData->nData}; + + if (pColData->flag != HAS_NULL) { + } + + if (taosArrayPush(aBlockCol, &blockCol) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } } - cksm = taosCalcChecksum(cksm, *pDiskData->ppKey, pDiskData->hdr.szUid + pDiskData->hdr.szVer + pDiskData->hdr.szKey); - n = taosWriteFile(pFD, *pDiskData->ppKey, pDiskData->hdr.szUid + pDiskData->hdr.szVer + pDiskData->hdr.szKey); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosWriteFile(pFD, &cksm, sizeof(cksm)); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - pBlkInfo->szKey = size + pDiskData->hdr.szUid + pDiskData->hdr.szVer + pDiskData->hdr.szKey + sizeof(TSCKSUM); - - // SBlockCol - if (pDiskData->hdr.szBlkCol == 0) { - pBlkInfo->szBlock = pBlkInfo->szKey; - goto _write_sma; - } - - code = tRealloc(ppBuf, pDiskData->hdr.szBlkCol + sizeof(TSCKSUM)); - if (code) goto _err; - - n = 0; - for (int32_t iBlockCol = 0; iBlockCol < taosArrayGetSize(pDiskData->aBlockCol); iBlockCol++) { - n += tPutBlockCol(*ppBuf + n, taosArrayGet(pDiskData->aBlockCol, iBlockCol)); - } - taosCalcChecksumAppend(0, *ppBuf, pDiskData->hdr.szBlkCol + sizeof(TSCKSUM)); - - n = taosWriteFile(pFD, *ppBuf, pDiskData->hdr.szBlkCol + sizeof(TSCKSUM)); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - for (int32_t iBlockCol = 0; iBlockCol < taosArrayGetSize(pDiskData->aBlockCol); iBlockCol++) { - } -#endif + // write // ================= SMA ==================== if (pSmaInfo) { diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 0ec922f20e..1b30a17dd2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1509,219 +1509,6 @@ int32_t tGetBlockData(uint8_t *p, SBlockData *pBlockData) { return n; } -// SDiskData ============================== -static int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t nOut, - int32_t *szOut, uint8_t **ppBuf) { - int32_t code = 0; - - ASSERT(szIn > 0 && ppOut); - - if (cmprAlg == NO_COMPRESSION) { - code = tRealloc(ppOut, nOut + szIn); - if (code) goto _exit; - - memcpy(*ppOut + nOut, pIn, szIn); - *szOut = szIn; - } else { - int32_t size = szIn + COMP_OVERFLOW_BYTES; - - code = tRealloc(ppOut, nOut + size); - if (code) goto _exit; - - if (cmprAlg == TWO_STAGE_COMP) { - ASSERT(ppBuf); - code = tRealloc(ppBuf, size); - if (code) goto _exit; - } - - *szOut = - tDataTypes[type].compFunc(pIn, szIn, szIn / tDataTypes[type].bytes, *ppOut + nOut, size, cmprAlg, *ppBuf, size); - if (*szOut <= 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _exit; - } - } - -_exit: - return code; -} - -static int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppBuf) { - int32_t code = 0; - - int32_t n = 0; - // bitmap - if (pColData->flag != HAS_VALUE) { - code = tsdbCmprData(pColData->pBitMap, BIT2_SIZE(pColData->nVal), TSDB_DATA_TYPE_TINYINT, cmprAlg, - pBlockCol->ppData, n, &pBlockCol->szBitmap, ppBuf); - if (code) goto _exit; - } else { - pBlockCol->szBitmap = 0; - } - n += pBlockCol->szBitmap; - - // offset - if (IS_VAR_DATA_TYPE(pColData->type)) { - code = tsdbCmprData((uint8_t *)pColData->aOffset, sizeof(int32_t) * pColData->nVal, TSDB_DATA_TYPE_INT, cmprAlg, - pBlockCol->ppData, n, &pBlockCol->szOffset, ppBuf); - if (code) goto _exit; - } else { - pBlockCol->szOffset = 0; - } - n += pBlockCol->szOffset; - - // value - if (pColData->flag != (HAS_NULL | HAS_NONE)) { - code = tsdbCmprData((uint8_t *)pColData->pData, pColData->nData, pColData->type, cmprAlg, pBlockCol->ppData, n, - &pBlockCol->szValue, ppBuf); - if (code) goto _exit; - } else { - pBlockCol->szValue = 0; - } - n += pBlockCol->szValue; - - // checksum - n += sizeof(TSCKSUM); - taosCalcChecksumAppend(0, *ppBuf, n); - -_exit: - return code; -} - -static int32_t tsdbDecmprData() { - int32_t code = 0; - // TODO - return code; -} - -int32_t tDiskDataInit(SDiskData *pDiskData) { - int32_t code = 0; - - pDiskData->aBlockCol = taosArrayInit(0, sizeof(SBlockCol)); - if (pDiskData->aBlockCol == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - pDiskData->aBuf = taosArrayInit(0, sizeof(uint8_t *)); - if (pDiskData->aBuf == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - -_exit: - return code; -} - -void tDiskDataClear(SDiskData *pDiskData) { - taosArrayDestroy(pDiskData->aBlockCol); - for (int32_t i = 0; i < taosArrayGetSize(pDiskData->aBuf); i++) { - tFree((uint8_t *)taosArrayGet(pDiskData->aBuf, i)); - } - taosArrayDestroy(pDiskData->aBuf); -} - -static uint8_t **tDiskDataAllocBuf(SDiskData *pDiskData) { - if (pDiskData->nBuf >= taosArrayGetSize(pDiskData->aBuf)) { - uint8_t *p = NULL; - if (taosArrayPush(pDiskData->aBuf, &p) == NULL) { - return NULL; - } - } - - ASSERT(pDiskData->nBuf < taosArrayGetSize(pDiskData->aBuf)); - uint8_t **pp = taosArrayGet(pDiskData->aBuf, pDiskData->nBuf); - pDiskData->nBuf++; - return pp; -} - -int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cmprAlg) { - int32_t code = 0; - - ASSERT(pBlockData->nRow > 0); - - pDiskData->cmprAlg = cmprAlg; - pDiskData->nRow = pBlockData->nRow; - pDiskData->suid = pBlockData->suid; - pDiskData->uid = pBlockData->uid; - pDiskData->szUid = 0; - pDiskData->szVer = 0; - pDiskData->szKey = 0; - taosArrayClear(pDiskData->aBlockCol); - pDiskData->nBuf = 0; - - { - pDiskData->ppKey = tDiskDataAllocBuf(pDiskData); - if (pDiskData->ppKey == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - int32_t n = 0; - // uid - if (pDiskData->uid == 0) { - code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, - cmprAlg, pDiskData->ppKey, n, &pDiskData->szUid, NULL); - if (code) goto _exit; - } else { - pDiskData->szUid = 0; - } - n += pDiskData->szUid; - - // version - code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, - cmprAlg, pDiskData->ppKey, n, &pDiskData->szVer, NULL); - if (code) goto _exit; - n += pDiskData->szVer; - - // tskey - code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, - cmprAlg, pDiskData->ppKey, &pDiskData->szKey, NULL); - if (code) goto _exit; - } - - // columns - int32_t offset = 0; - for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - - if (pColData->flag == HAS_NONE) continue; - - SBlockCol blockCol = {.cid = pColData->cid, - .type = pColData->type, - .smaOn = pColData->smaOn, - .flag = pColData->flag, - .szOrigin = pColData->nData}; - - if (pColData->flag != HAS_NULL) { - // alloc a buffer - blockCol.ppData = tDiskDataAllocBuf(pDiskData); - if (blockCol.ppData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - // compress - code = tsdbCmprColData(pColData, cmprAlg, &blockCol, NULL); - if (code) goto _exit; - - // update offset - blockCol.offset = offset; - offset = offset + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue; - } - - if (taosArrayPush(pDiskData->aBlockCol, &blockCol) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - } - -_exit: - return code; -} -int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cmprAlg); -int32_t tDiskToBlockData(SDiskData *pDiskData, SBlockData *pBlockData); - // SDiskDataHdr ============================== int32_t tPutDiskDataHdr(uint8_t *p, void *ph) { int32_t n = 0; @@ -1927,3 +1714,168 @@ void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { } } } + +int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t nOut, + int32_t *szOut, uint8_t **ppBuf) { + int32_t code = 0; + + ASSERT(szIn > 0 && ppOut); + + if (cmprAlg == NO_COMPRESSION) { + code = tRealloc(ppOut, nOut + szIn); + if (code) goto _exit; + + memcpy(*ppOut + nOut, pIn, szIn); + *szOut = szIn; + } else { + int32_t size = szIn + COMP_OVERFLOW_BYTES; + + code = tRealloc(ppOut, nOut + size); + if (code) goto _exit; + + if (cmprAlg == TWO_STAGE_COMP) { + ASSERT(ppBuf); + code = tRealloc(ppBuf, size); + if (code) goto _exit; + } + + *szOut = + tDataTypes[type].compFunc(pIn, szIn, szIn / tDataTypes[type].bytes, *ppOut + nOut, size, cmprAlg, *ppBuf, size); + if (*szOut <= 0) { + code = TSDB_CODE_COMPRESS_ERROR; + goto _exit; + } + } + +_exit: + return code; +} + +int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppBuf) { + int32_t code = 0; + + int32_t n = 0; + // bitmap + if (pColData->flag != HAS_VALUE) { + code = tsdbCmprData(pColData->pBitMap, BIT2_SIZE(pColData->nVal), TSDB_DATA_TYPE_TINYINT, cmprAlg, + pBlockCol->ppData, n, &pBlockCol->szBitmap, ppBuf); + if (code) goto _exit; + } else { + pBlockCol->szBitmap = 0; + } + n += pBlockCol->szBitmap; + + // offset + if (IS_VAR_DATA_TYPE(pColData->type)) { + code = tsdbCmprData((uint8_t *)pColData->aOffset, sizeof(int32_t) * pColData->nVal, TSDB_DATA_TYPE_INT, cmprAlg, + pBlockCol->ppData, n, &pBlockCol->szOffset, ppBuf); + if (code) goto _exit; + } else { + pBlockCol->szOffset = 0; + } + n += pBlockCol->szOffset; + + // value + if (pColData->flag != (HAS_NULL | HAS_NONE)) { + code = tsdbCmprData((uint8_t *)pColData->pData, pColData->nData, pColData->type, cmprAlg, pBlockCol->ppData, n, + &pBlockCol->szValue, ppBuf); + if (code) goto _exit; + } else { + pBlockCol->szValue = 0; + } + n += pBlockCol->szValue; + + // checksum + n += sizeof(TSCKSUM); + taosCalcChecksumAppend(0, *ppBuf, n); + +_exit: + return code; +} + +#if 0 +int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cmprAlg) { + int32_t code = 0; + + ASSERT(pBlockData->nRow > 0); + + pDiskData->cmprAlg = cmprAlg; + pDiskData->nRow = pBlockData->nRow; + pDiskData->suid = pBlockData->suid; + pDiskData->uid = pBlockData->uid; + pDiskData->szUid = 0; + pDiskData->szVer = 0; + pDiskData->szKey = 0; + taosArrayClear(pDiskData->aBlockCol); + pDiskData->nBuf = 0; + + { + pDiskData->ppKey = tDiskDataAllocBuf(pDiskData); + if (pDiskData->ppKey == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + int32_t n = 0; + // uid + if (pDiskData->uid == 0) { + code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, + cmprAlg, pDiskData->ppKey, n, &pDiskData->szUid, NULL); + if (code) goto _exit; + } else { + pDiskData->szUid = 0; + } + n += pDiskData->szUid; + + // version + code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, + cmprAlg, pDiskData->ppKey, n, &pDiskData->szVer, NULL); + if (code) goto _exit; + n += pDiskData->szVer; + + // tskey + code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, + cmprAlg, pDiskData->ppKey, &pDiskData->szKey, NULL); + if (code) goto _exit; + } + + // columns + int32_t offset = 0; + for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { + SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); + + if (pColData->flag == HAS_NONE) continue; + + SBlockCol blockCol = {.cid = pColData->cid, + .type = pColData->type, + .smaOn = pColData->smaOn, + .flag = pColData->flag, + .szOrigin = pColData->nData}; + + if (pColData->flag != HAS_NULL) { + // alloc a buffer + blockCol.ppData = tDiskDataAllocBuf(pDiskData); + if (blockCol.ppData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + // compress + code = tsdbCmprColData(pColData, cmprAlg, &blockCol, NULL); + if (code) goto _exit; + + // update offset + blockCol.offset = offset; + offset = offset + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue; + } + + if (taosArrayPush(pDiskData->aBlockCol, &blockCol) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + } + +_exit: + return code; +} +#endif From 2d503135864431cd143859cb60e7d6dec26895be Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 7 Aug 2022 08:12:59 +0000 Subject: [PATCH 22/92] more work --- source/dnode/vnode/src/inc/tsdb.h | 9 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 89 ++++++++++++++----- source/dnode/vnode/src/tsdb/tsdbUtil.c | 43 ++++----- 3 files changed, 95 insertions(+), 46 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index fe3992102b..f746da3857 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -63,6 +63,7 @@ typedef struct SRowMerger SRowMerger; typedef struct STsdbReadSnap STsdbReadSnap; typedef struct SBlockInfo SBlockInfo; typedef struct SSmaInfo SSmaInfo; +typedef struct SBlockCol SBlockCol; #define TSDB_MAX_SUBBLOCKS 8 #define TSDB_FHDR_SIZE 512 @@ -181,7 +182,8 @@ int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t nOut, int32_t *szOut, uint8_t **ppBuf); -int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppBuf); +int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppOut, int8_t nOut, + uint8_t **ppBuf); // tsdbMemTable ============================================================================================== // SMemTable int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); @@ -400,7 +402,7 @@ struct SMapData { uint8_t *pData; }; -typedef struct { +struct SBlockCol { int16_t cid; int8_t type; int8_t smaOn; @@ -410,7 +412,7 @@ typedef struct { int32_t szOffset; // offset size, 0 only for non-variant-length type int32_t szValue; // value size, 0 when flag == (HAS_NULL | HAS_NONE) int32_t offset; -} SBlockCol; +}; struct SBlockInfo { int64_t offset; // block data offset @@ -600,6 +602,7 @@ struct SDataFWriter { uint8_t *pBuf1; uint8_t *pBuf2; + uint8_t *pBuf3; }; struct STsdbReadSnap { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 31029e6bb4..416b35aa03 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1414,7 +1414,6 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - code = tDiskDataInit(&pWriter->dData); if (code) goto _err; pWriter->pTsdb = pTsdb; pWriter->wSet = (SDFileSet){.diskId = pSet->diskId, @@ -1594,9 +1593,9 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { goto _err; } - tDiskDataClear(&(*ppWriter)->dData); tFree((*ppWriter)->pBuf1); tFree((*ppWriter)->pBuf2); + tFree((*ppWriter)->pBuf3); taosMemoryFree(*ppWriter); _exit: *ppWriter = NULL; @@ -1905,11 +1904,14 @@ _err: int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, int8_t cmprAlg, int8_t toLast) { - int32_t code = 0; - TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; + int32_t code = 0; ASSERT(pBlockData->nRow > 0); + pBlkInfo->offset = toLast ? pWriter->fLast.size : pWriter->fData.size; + pBlkInfo->szBlock = 0; + pBlkInfo->szKey = 0; + // ================= DATA ==================== SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pBlockData->suid, @@ -1923,23 +1925,8 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock goto _err; } - // uid - if (pBlockData->uid == 0) { - ASSERT(toLast); - code = tsdbCmprData(); - if (code) goto _err; - } - - // version - code = tsdbCmprData(); - if (code) goto _err; - - // ts - code = tsdbCmprData(); - if (code) goto _err; - - // columns - int32_t offset = 0; + // encode ================= + int32_t nBuf1 = 0; for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); @@ -1954,15 +1941,71 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock .szOrigin = pColData->nData}; if (pColData->flag != HAS_NULL) { + code = tsdbCmprColData(pColData, cmprAlg, &blockCol, &pWriter->pBuf1, nBuf1, &pWriter->pBuf3); + if (code) goto _err; + + blockCol.offset = nBuf1; + nBuf1 = nBuf1 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); } if (taosArrayPush(aBlockCol, &blockCol) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } + + hdr.szBlkCol += tPutBlockCol(NULL, &blockCol); } - // write + // (uid + version + tskey + aBlockCol) + if (pBlockData->uid == 0) { + code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, + &pWriter->pBuf2, 0, &hdr.szUid, &pWriter->pBuf3); + if (code) goto _err; + } + + code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, + cmprAlg, &pWriter->pBuf2, hdr.szUid, &hdr.szVer, &pWriter->pBuf3); + if (code) goto _err; + + code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, + cmprAlg, &pWriter->pBuf2, hdr.szUid + hdr.szVer, &hdr.szKey, &pWriter->pBuf3); + if (code) goto _err; + + pBlkInfo->szKey = tPutDiskDataHdr(NULL, &hdr); + code = tRealloc(&pWriter->pBuf3, pBlkInfo->szKey); + if (code) goto _err; + tPutDiskDataHdr(pWriter->pBuf3, &hdr); + TSCKSUM cksm = taosCalcChecksum(0, pWriter->pBuf3, pBlkInfo->szKey); + + // write ================= + TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; + + // hdr + int64_t n = taosWriteFile(pFD, pWriter->pBuf3, pBlkInfo->szKey); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // uid + version + tskey + (CKSM) + taosCalcChecksumAppend(cksm, pWriter->pBuf2, hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM)); + n = taosWriteFile(pFD, pWriter->pBuf2, hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM)); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + pBlkInfo->szKey = pBlkInfo->szKey + hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM); + + // aBlockCol + + // colmns + + // update info + if (toLast) { + pWriter->fLast.size += pBlkInfo->szBlock; + } else { + pWriter->fData.size += pBlkInfo->szBlock; + } // ================= SMA ==================== if (pSmaInfo) { @@ -1971,10 +2014,12 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock } _exit: + taosArrayDestroy(aBlockCol); return code; _err: tsdbError("vgId:%d tsdb write block data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + taosArrayDestroy(aBlockCol); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 1b30a17dd2..8f3bb6e89c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1373,11 +1373,11 @@ int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlock c = tsdbKeyCmprFn(&TSDBROW_KEY(&row1), &TSDBROW_KEY(&row2)); if (c < 0) { - code = tBlockDataAppendRow(pBlockData, &row1, NULL); + // code = tBlockDataAppendRow(pBlockData, &row1, NULL); if (code) goto _exit; iRow1++; } else if (c > 0) { - code = tBlockDataAppendRow(pBlockData, &row2, NULL); + // code = tBlockDataAppendRow(pBlockData, &row2, NULL); if (code) goto _exit; iRow2++; } else { @@ -1387,14 +1387,14 @@ int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlock while (iRow1 < nRow1) { row1 = tsdbRowFromBlockData(pBlockData1, iRow1); - code = tBlockDataAppendRow(pBlockData, &row1, NULL); + // code = tBlockDataAppendRow(pBlockData, &row1, NULL); if (code) goto _exit; iRow1++; } while (iRow2 < nRow2) { row2 = tsdbRowFromBlockData(pBlockData2, iRow2); - code = tBlockDataAppendRow(pBlockData, &row2, NULL); + // code = tBlockDataAppendRow(pBlockData, &row2, NULL); if (code) goto _exit; iRow2++; } @@ -1751,43 +1751,44 @@ _exit: return code; } -int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppBuf) { +int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppOut, int8_t nOut, + uint8_t **ppBuf) { int32_t code = 0; - int32_t n = 0; + pBlockCol->szBitmap = 0; + pBlockCol->szOffset = 0; + pBlockCol->szValue = 0; + + int32_t size = 0; // bitmap if (pColData->flag != HAS_VALUE) { - code = tsdbCmprData(pColData->pBitMap, BIT2_SIZE(pColData->nVal), TSDB_DATA_TYPE_TINYINT, cmprAlg, - pBlockCol->ppData, n, &pBlockCol->szBitmap, ppBuf); + code = tsdbCmprData(pColData->pBitMap, BIT2_SIZE(pColData->nVal), TSDB_DATA_TYPE_TINYINT, cmprAlg, ppOut, + nOut + size, &pBlockCol->szBitmap, ppBuf); if (code) goto _exit; - } else { - pBlockCol->szBitmap = 0; } - n += pBlockCol->szBitmap; + size += pBlockCol->szBitmap; // offset if (IS_VAR_DATA_TYPE(pColData->type)) { code = tsdbCmprData((uint8_t *)pColData->aOffset, sizeof(int32_t) * pColData->nVal, TSDB_DATA_TYPE_INT, cmprAlg, - pBlockCol->ppData, n, &pBlockCol->szOffset, ppBuf); + ppOut, nOut + size, &pBlockCol->szOffset, ppBuf); if (code) goto _exit; - } else { - pBlockCol->szOffset = 0; } - n += pBlockCol->szOffset; + size += pBlockCol->szOffset; // value if (pColData->flag != (HAS_NULL | HAS_NONE)) { - code = tsdbCmprData((uint8_t *)pColData->pData, pColData->nData, pColData->type, cmprAlg, pBlockCol->ppData, n, + code = tsdbCmprData((uint8_t *)pColData->pData, pColData->nData, pColData->type, cmprAlg, ppOut, nOut + size, &pBlockCol->szValue, ppBuf); if (code) goto _exit; - } else { - pBlockCol->szValue = 0; } - n += pBlockCol->szValue; + size += pBlockCol->szValue; // checksum - n += sizeof(TSCKSUM); - taosCalcChecksumAppend(0, *ppBuf, n); + size += sizeof(TSCKSUM); + code = tRealloc(ppOut, nOut + size); + if (code) goto _exit; + taosCalcChecksumAppend(0, *ppOut + nOut, size); _exit: return code; From 94069ec20a1c20328986d888cc052ded5378d795 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 7 Aug 2022 08:35:57 +0000 Subject: [PATCH 23/92] more work --- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 38 +++++++- source/dnode/vnode/src/tsdb/tsdbUtil.c | 87 ------------------- 2 files changed, 36 insertions(+), 89 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 416b35aa03..1d5625220d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1926,6 +1926,7 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock } // encode ================= + // columns int32_t nBuf1 = 0; for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); @@ -1956,27 +1957,34 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock hdr.szBlkCol += tPutBlockCol(NULL, &blockCol); } - // (uid + version + tskey + aBlockCol) + // uid if (pBlockData->uid == 0) { code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, &pWriter->pBuf2, 0, &hdr.szUid, &pWriter->pBuf3); if (code) goto _err; } + // version code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, &pWriter->pBuf2, hdr.szUid, &hdr.szVer, &pWriter->pBuf3); if (code) goto _err; + // tskey code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg, &pWriter->pBuf2, hdr.szUid + hdr.szVer, &hdr.szKey, &pWriter->pBuf3); if (code) goto _err; + // hdr pBlkInfo->szKey = tPutDiskDataHdr(NULL, &hdr); code = tRealloc(&pWriter->pBuf3, pBlkInfo->szKey); if (code) goto _err; tPutDiskDataHdr(pWriter->pBuf3, &hdr); TSCKSUM cksm = taosCalcChecksum(0, pWriter->pBuf3, pBlkInfo->szKey); + code = tRealloc(&pWriter->pBuf2, hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM)); + if (code) goto _err; + taosCalcChecksumAppend(cksm, pWriter->pBuf2, hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM)); + // write ================= TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; @@ -1988,17 +1996,43 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock } // uid + version + tskey + (CKSM) - taosCalcChecksumAppend(cksm, pWriter->pBuf2, hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM)); n = taosWriteFile(pFD, pWriter->pBuf2, hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM)); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } pBlkInfo->szKey = pBlkInfo->szKey + hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM); + pBlkInfo->szBlock += pBlkInfo->szKey; // aBlockCol + if (hdr.szBlkCol > 0) { + code = tRealloc(&pWriter->pBuf2, hdr.szBlkCol + sizeof(TSCKSUM)); + if (code) goto _err; + + n = 0; + for (int32_t iBlockCol = 0; iBlockCol < taosArrayGetSize(aBlockCol); iBlockCol++) { + n += tPutBlockCol(pWriter->pBuf2 + n, taosArrayGet(aBlockCol, iBlockCol)); + } + ASSERT(n == hdr.szBlkCol); + taosCalcChecksumAppend(0, pWriter->pBuf2, hdr.szBlkCol + sizeof(TSCKSUM)); + + n = taosWriteFile(pFD, pWriter->pBuf2, hdr.szBlkCol + sizeof(TSCKSUM)); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + } // colmns + if (nBuf1 > 0) { + n = taosWriteFile(pFD, pWriter->pBuf1, nBuf1); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + pBlkInfo->szBlock += nBuf1; + } // update info if (toLast) { diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 8f3bb6e89c..d669c28960 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1793,90 +1793,3 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol _exit: return code; } - -#if 0 -int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cmprAlg) { - int32_t code = 0; - - ASSERT(pBlockData->nRow > 0); - - pDiskData->cmprAlg = cmprAlg; - pDiskData->nRow = pBlockData->nRow; - pDiskData->suid = pBlockData->suid; - pDiskData->uid = pBlockData->uid; - pDiskData->szUid = 0; - pDiskData->szVer = 0; - pDiskData->szKey = 0; - taosArrayClear(pDiskData->aBlockCol); - pDiskData->nBuf = 0; - - { - pDiskData->ppKey = tDiskDataAllocBuf(pDiskData); - if (pDiskData->ppKey == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - int32_t n = 0; - // uid - if (pDiskData->uid == 0) { - code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, - cmprAlg, pDiskData->ppKey, n, &pDiskData->szUid, NULL); - if (code) goto _exit; - } else { - pDiskData->szUid = 0; - } - n += pDiskData->szUid; - - // version - code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, - cmprAlg, pDiskData->ppKey, n, &pDiskData->szVer, NULL); - if (code) goto _exit; - n += pDiskData->szVer; - - // tskey - code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, - cmprAlg, pDiskData->ppKey, &pDiskData->szKey, NULL); - if (code) goto _exit; - } - - // columns - int32_t offset = 0; - for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - - if (pColData->flag == HAS_NONE) continue; - - SBlockCol blockCol = {.cid = pColData->cid, - .type = pColData->type, - .smaOn = pColData->smaOn, - .flag = pColData->flag, - .szOrigin = pColData->nData}; - - if (pColData->flag != HAS_NULL) { - // alloc a buffer - blockCol.ppData = tDiskDataAllocBuf(pDiskData); - if (blockCol.ppData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - // compress - code = tsdbCmprColData(pColData, cmprAlg, &blockCol, NULL); - if (code) goto _exit; - - // update offset - blockCol.offset = offset; - offset = offset + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue; - } - - if (taosArrayPush(pDiskData->aBlockCol, &blockCol) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - } - -_exit: - return code; -} -#endif From 3698016ad9ee5d62ec7cbfabdb3e76970cb6957b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 7 Aug 2022 09:25:50 +0000 Subject: [PATCH 24/92] more work --- source/dnode/vnode/src/inc/tsdb.h | 1 + .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 91 ++++++++----------- 2 files changed, 40 insertions(+), 52 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index f746da3857..5bbc0d45f1 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -603,6 +603,7 @@ struct SDataFWriter { uint8_t *pBuf1; uint8_t *pBuf2; uint8_t *pBuf3; + uint8_t *pBuf4; }; struct STsdbReadSnap { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 1d5625220d..faabf60f71 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1596,6 +1596,7 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { tFree((*ppWriter)->pBuf1); tFree((*ppWriter)->pBuf2); tFree((*ppWriter)->pBuf3); + tFree((*ppWriter)->pBuf4); taosMemoryFree(*ppWriter); _exit: *ppWriter = NULL; @@ -1919,14 +1920,8 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock .nRow = pBlockData->nRow, .cmprAlg = cmprAlg}; - SArray *aBlockCol = taosArrayInit(taosArrayGetSize(pBlockData->aIdx), sizeof(SBlockCol)); - if (aBlockCol == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - // encode ================= - // columns + // columns AND SBlockCol int32_t nBuf1 = 0; for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); @@ -1949,89 +1944,83 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock nBuf1 = nBuf1 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); } - if (taosArrayPush(aBlockCol, &blockCol) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - - hdr.szBlkCol += tPutBlockCol(NULL, &blockCol); + code = tRealloc(&pWriter->pBuf2, hdr.szBlkCol + tPutBlockCol(NULL, &blockCol)); + if (code) goto _err; + hdr.szBlkCol += tPutBlockCol(pWriter->pBuf2 + hdr.szBlkCol, &blockCol); } - // uid + int32_t nBuf2 = 0; + if (hdr.szBlkCol > 0) { + nBuf2 = hdr.szBlkCol + sizeof(TSCKSUM); + + code = tRealloc(&pWriter->pBuf2, nBuf2); + if (code) goto _err; + + taosCalcChecksumAppend(0, pWriter->pBuf2, nBuf2); + } + + // uid + version + tskey + int32_t nBuf3 = 0; if (pBlockData->uid == 0) { code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, - &pWriter->pBuf2, 0, &hdr.szUid, &pWriter->pBuf3); + &pWriter->pBuf3, nBuf3, &hdr.szUid, &pWriter->pBuf4); if (code) goto _err; } + nBuf3 += hdr.szUid; - // version code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, - cmprAlg, &pWriter->pBuf2, hdr.szUid, &hdr.szVer, &pWriter->pBuf3); + cmprAlg, &pWriter->pBuf3, nBuf3, &hdr.szVer, &pWriter->pBuf4); if (code) goto _err; + nBuf3 += hdr.szVer; - // tskey code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, - cmprAlg, &pWriter->pBuf2, hdr.szUid + hdr.szVer, &hdr.szKey, &pWriter->pBuf3); + cmprAlg, &pWriter->pBuf3, nBuf3, &hdr.szKey, &pWriter->pBuf4); + if (code) goto _err; + nBuf3 += hdr.szKey; + + nBuf3 += sizeof(TSCKSUM); + code = tRealloc(&pWriter->pBuf3, nBuf3); if (code) goto _err; // hdr - pBlkInfo->szKey = tPutDiskDataHdr(NULL, &hdr); - code = tRealloc(&pWriter->pBuf3, pBlkInfo->szKey); + int32_t nBuf4 = tPutDiskDataHdr(NULL, &hdr); + code = tRealloc(&pWriter->pBuf4, nBuf4); if (code) goto _err; - tPutDiskDataHdr(pWriter->pBuf3, &hdr); - TSCKSUM cksm = taosCalcChecksum(0, pWriter->pBuf3, pBlkInfo->szKey); - - code = tRealloc(&pWriter->pBuf2, hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM)); - if (code) goto _err; - taosCalcChecksumAppend(cksm, pWriter->pBuf2, hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM)); + tPutDiskDataHdr(pWriter->pBuf4, &hdr); + taosCalcChecksumAppend(taosCalcChecksum(0, pWriter->pBuf4, nBuf4), pWriter->pBuf3, nBuf3); // write ================= TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; - // hdr - int64_t n = taosWriteFile(pFD, pWriter->pBuf3, pBlkInfo->szKey); + pBlkInfo->szKey = nBuf4 + nBuf3; + pBlkInfo->szBlock = nBuf1 + nBuf2 + nBuf3 + nBuf4; + + int64_t n = taosWriteFile(pFD, pWriter->pBuf4, nBuf4); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - // uid + version + tskey + (CKSM) - n = taosWriteFile(pFD, pWriter->pBuf2, hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM)); + n = taosWriteFile(pFD, pWriter->pBuf3, nBuf3); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - pBlkInfo->szKey = pBlkInfo->szKey + hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM); - pBlkInfo->szBlock += pBlkInfo->szKey; - // aBlockCol - if (hdr.szBlkCol > 0) { - code = tRealloc(&pWriter->pBuf2, hdr.szBlkCol + sizeof(TSCKSUM)); - if (code) goto _err; - - n = 0; - for (int32_t iBlockCol = 0; iBlockCol < taosArrayGetSize(aBlockCol); iBlockCol++) { - n += tPutBlockCol(pWriter->pBuf2 + n, taosArrayGet(aBlockCol, iBlockCol)); - } - ASSERT(n == hdr.szBlkCol); - taosCalcChecksumAppend(0, pWriter->pBuf2, hdr.szBlkCol + sizeof(TSCKSUM)); - - n = taosWriteFile(pFD, pWriter->pBuf2, hdr.szBlkCol + sizeof(TSCKSUM)); + if (nBuf2) { + n = taosWriteFile(pFD, pWriter->pBuf2, nBuf2); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } } - // colmns - if (nBuf1 > 0) { + if (nBuf1) { n = taosWriteFile(pFD, pWriter->pBuf1, nBuf1); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - - pBlkInfo->szBlock += nBuf1; } // update info @@ -2048,12 +2037,10 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock } _exit: - taosArrayDestroy(aBlockCol); return code; _err: tsdbError("vgId:%d tsdb write block data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - taosArrayDestroy(aBlockCol); return code; } From 605e89b6a17156d82642b1c071e467213b750760 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 7 Aug 2022 12:50:42 +0000 Subject: [PATCH 25/92] more work --- source/dnode/vnode/src/inc/tsdb.h | 6 + .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 202 +++++++++++++----- source/dnode/vnode/src/tsdb/tsdbUtil.c | 75 +++++++ 3 files changed, 235 insertions(+), 48 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 5bbc0d45f1..a824f8f0b3 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -182,8 +182,12 @@ int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t nOut, int32_t *szOut, uint8_t **ppBuf); +int32_t tsdbDecmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t szOut, + uint8_t **ppBuf); int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppOut, int8_t nOut, uint8_t **ppBuf); +int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, int32_t nVal, SColData *pColData, + uint8_t **ppBuf); // tsdbMemTable ============================================================================================== // SMemTable int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); @@ -254,6 +258,7 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMa int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL); int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg); +#if 0 int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int16_t *aColId, int32_t nCol, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, SBlockData *pBlockData, @@ -262,6 +267,7 @@ int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBl uint8_t **ppBuf2); int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); +#endif // SDelFWriter int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index faabf60f71..125e1bcb73 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -677,6 +677,159 @@ _err: return code; } +int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg) { + int32_t code = 0; + SSmaInfo *pSmaInfo = &pBlock->smaInfo; + + ASSERT(pSmaInfo->size > 0); + + taosArrayClear(aColumnDataAgg); + + // alloc + int32_t size = pSmaInfo->size + sizeof(TSCKSUM); + code = tRealloc(&pReader->pBuf1, size); + if (code) goto _err; + + // read + int64_t n = taosReadFile(pReader->pSmaFD, pReader->pBuf1, size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } else if (n < size) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + + // check + if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + + // decode + n = 0; + while (n < pSmaInfo->size) { + SColumnDataAgg sma; + + n += tGetColumnDataAgg(pReader->pBuf1 + n, &sma); + if (taosArrayPush(aColumnDataAgg, &sma) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + } + + return code; + +_err: + tsdbError("vgId:%d tsdb read block sma failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + return code; +} + +static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo, int8_t fromLast, int16_t *aColId, + int32_t nColId, SBlockData *pBlockData) { + int32_t code = 0; + + tBlockDataReset(pBlockData); + + TdFilePtr pFD = fromLast ? pReader->pLastFD : pReader->pDataFD; + + // seek + int64_t n = taosLSeekFile(pFD, pBlkInfo->offset, SEEK_SET); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // read + code = tRealloc(&pReader->pBuf1, pBlkInfo->szBlock); + if (code) goto _err; + + n = taosReadFile(pFD, pReader->pBuf1, pBlkInfo->szBlock); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } else if (n < pBlkInfo->szBlock) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + + uint8_t *p = pReader->pBuf1; + // check & decode + SDiskDataHdr hdr; + if (!taosCheckChecksumWhole(p, pBlkInfo->szKey)) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + p += tGetDiskDataHdr(p, &hdr); + + tBlockDataSetSchema(pBlockData, NULL, hdr.suid, hdr.uid); + pBlockData->nRow = hdr.nRow; + + if (hdr.uid == 0) { + ASSERT(hdr.szUid); + code = tsdbDecmprData(p, hdr.szUid, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aUid, + sizeof(int64_t) * hdr.nRow, &pReader->pBuf2); + if (code) goto _err; + } else { + ASSERT(hdr.szUid == 0); + } + p += hdr.szUid; + + code = tsdbDecmprData(p, hdr.szVer, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aVersion, + sizeof(int64_t) * hdr.nRow, &pReader->pBuf2); + if (code) goto _err; + p += hdr.szVer; + + code = tsdbDecmprData(p, hdr.szKey, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aTSKEY, + sizeof(TSKEY) * hdr.nRow, &pReader->pBuf2); + if (code) goto _err; + p += hdr.szKey; + p += sizeof(TSCKSUM); + + // SBlockCol + if (hdr.szBlkCol > 0) { + if (!taosCheckChecksumWhole(p, hdr.szBlkCol + sizeof(TSCKSUM))) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + + int32_t iColData = 0; + uint8_t *pt = p + hdr.szBlkCol + sizeof(TSCKSUM); + n = 0; + while (n < hdr.szBlkCol) { + SBlockCol blockCol; + + n += tGetBlockCol(p + n, &blockCol); + + ASSERT(blockCol.flag && blockCol.flag != HAS_NONE); + + SColData *pColData; + code = tBlockDataAddColData(pBlockData, iColData, &pColData); + if (code) goto _err; + iColData++; + + tColDataInit(pColData, blockCol.cid, blockCol.type, blockCol.smaOn); + + if (blockCol.flag == HAS_NULL) { + for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { + code = tColDataAppendValue(pColData, &COL_VAL_NULL(blockCol.cid, blockCol.type)); + if (code) goto _err; + } + } else { + code = tsdbDecmprColData(pt + blockCol.offset, &blockCol, hdr.cmprAlg, hdr.nRow, pColData, pReader->pBuf2); + if (code) goto _err; + } + } + } + + return code; + +_err: + tsdbError("vgId:%d tsdb read block data impl failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + return code; +} + +#if 0 static int32_t tsdbReadBlockDataKey(SBlockData *pBlockData, SBlockInfo *pSubBlock, uint8_t *pBuf, uint8_t **ppBuf) { int32_t code = 0; #if 0 @@ -1350,54 +1503,7 @@ _err: #endif return code; } - -int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg) { - int32_t code = 0; - SSmaInfo *pSmaInfo = &pBlock->smaInfo; - - ASSERT(pSmaInfo->size > 0); - - taosArrayClear(aColumnDataAgg); - - // alloc - int32_t size = pSmaInfo->size + sizeof(TSCKSUM); - code = tRealloc(&pReader->pBuf1, size); - if (code) goto _err; - - // read - int64_t n = taosReadFile(pReader->pSmaFD, pReader->pBuf1, size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // check - if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // decode - n = 0; - while (n < pSmaInfo->size) { - SColumnDataAgg sma; - - n += tGetColumnDataAgg(pReader->pBuf1 + n, &sma); - if (taosArrayPush(aColumnDataAgg, &sma) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - } - - return code; - -_err: - tsdbError("vgId:%d tsdb read block sma failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); - return code; -} +#endif // SDataFWriter ==================================================== int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet) { diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index d669c28960..dbb35532e8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1751,6 +1751,36 @@ _exit: return code; } +int32_t tsdbDecmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t szOut, + uint8_t **ppBuf) { + int32_t code = 0; + + code = tRealloc(ppOut, szOut); + if (code) goto _exit; + + if (cmprAlg == NO_COMPRESSION) { + ASSERT(szIn == szOut); + memcpy(*ppOut, pIn, szOut); + } else { + if (cmprAlg == TWO_STAGE_COMP) { + code = tRealloc(ppBuf, szOut + COMP_OVERFLOW_BYTES); + if (code) goto _exit; + } + + int32_t size = tDataTypes[type].decompFunc(pIn, szIn, szOut / tDataTypes[type].bytes, *ppOut, szOut, cmprAlg, + *ppBuf, szOut + COMP_OVERFLOW_BYTES); + if (size <= 0) { + code = TSDB_CODE_COMPRESS_ERROR; + goto _exit; + } + + ASSERT(size == szOut); + } + +_exit: + return code; +} + int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppOut, int8_t nOut, uint8_t **ppBuf) { int32_t code = 0; @@ -1793,3 +1823,48 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol _exit: return code; } + +int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, int32_t nVal, SColData *pColData, + uint8_t **ppBuf) { + int32_t code = 0; + + int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM); + if (!taosCheckChecksumWhole(pIn, size)) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } + + pColData->cid = pBlockCol->cid; + pColData->type = pBlockCol->type; + pColData->smaOn = pBlockCol->smaOn; + pColData->flag = pBlockCol->flag; + pColData->nVal = nVal; + pColData->nData = pBlockCol->szOrigin; + + uint8_t *p = pIn; + // bitmap + if (pBlockCol->szBitmap) { + code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pColData->pBitMap, + BIT2_SIZE(pColData->nVal), ppBuf); + if (code) goto _exit; + } + p += pBlockCol->szBitmap; + + // offset + if (pBlockCol->szOffset) { + code = tsdbDecmprData(p, pBlockCol->szOffset, TSDB_DATA_TYPE_INT, cmprAlg, (uint8_t **)&pColData->aOffset, + sizeof(int32_t) * pColData->nVal, ppBuf); + if (code) goto _exit; + } + p += pBlockCol->szOffset; + + // value + if (pBlockCol->szValue) { + code = tsdbDecmprData(p, pBlockCol->szValue, pColData->type, cmprAlg, &pColData->pData, pColData->nData, ppBuf); + if (code) goto _exit; + } + p += pBlockCol->szValue; + +_exit: + return code; +} \ No newline at end of file From 71691eb7daaa38bd9a3852444ed5103c66da326f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 7 Aug 2022 13:14:43 +0000 Subject: [PATCH 26/92] make code compile --- source/dnode/vnode/src/inc/tsdb.h | 4 +++ source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 4 +-- source/dnode/vnode/src/tsdb/tsdbRead.c | 3 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 35 ++++++++++++++++++- 5 files changed, 42 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index a824f8f0b3..6af1a1a79b 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -257,6 +257,10 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx); int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData); int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL); int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg); +int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, int16_t *aColId, + int32_t nColId); +int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, int16_t *aColId, + int32_t nColId); #if 0 int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int16_t *aColId, int32_t nCol, diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 707aa3153d..c2bfad264b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -507,7 +507,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetBlock); /* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); */ - code = tsdbReadBlockData(state->pDataFReader, state->pBlockIdx, &block, state->pBlockData, NULL, NULL); + code = tsdbReadDataBlock(state->pDataFReader, &block, state->pBlockData, NULL, 0); if (code) goto _err; state->nRow = state->blockData.nRow; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 4269a785a0..ad721ede7a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -309,7 +309,7 @@ static int32_t tsdbCommitterNextLastRow(SCommitter *pCommitter) { pCommitter->dReader.iBlockL++; if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { pCommitter->dReader.pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); - code = tsdbReadLastBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockL, pBlockDatal, NULL, NULL); + code = tsdbReadLastBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockL, pBlockDatal, NULL, 0); if (code) goto _exit; pCommitter->dReader.iRow = 0; @@ -593,7 +593,7 @@ static int32_t tsdbMergeCommitData(SCommitter *pCommitter, STbDataIter *pIter, S SBlockData *pBlockDataR = &pCommitter->dReader.bData; SBlockData *pBlockDataW = &pCommitter->dWriter.bData; - code = tsdbReadDataBlock(pCommitter->dReader.pReader, pBlock, pBlockDataR, NULL, NULL); + code = tsdbReadDataBlock(pCommitter->dReader.pReader, pBlock, pBlockDataR, NULL, 0); if (code) goto _err; tBlockDataClearData(pBlockDataW); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 87cc000908..0157f0ed22 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -785,8 +785,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBlockIdx blockIdx = {.suid = pReader->suid, .uid = pBlockScanInfo->uid}; - int32_t code = - tsdbReadColData(pReader->pFileReader, &blockIdx, pBlock, pSupInfo->colIds, numOfCols, pBlockData, NULL, NULL); + int32_t code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData, pSupInfo->colIds, numOfCols); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 125e1bcb73..4f02299c7e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -816,7 +816,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo if (code) goto _err; } } else { - code = tsdbDecmprColData(pt + blockCol.offset, &blockCol, hdr.cmprAlg, hdr.nRow, pColData, pReader->pBuf2); + code = tsdbDecmprColData(pt + blockCol.offset, &blockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->pBuf2); if (code) goto _err; } } @@ -829,6 +829,39 @@ _err: return code; } +int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, int16_t *aColId, + int32_t nColId) { + int32_t code = 0; + + code = tsdbReadBlockDataImpl(pReader, &pBlock->aSubBlock[0], 0, aColId, nColId, pBlockData); + if (code) goto _err; + + for (int32_t iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { + // TODO + ASSERT(0); + } + + return code; + +_err: + tsdbError("vgId:%d tsdb read data block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + return code; +} + +int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, int16_t *aColId, + int32_t nColId) { + int32_t code = 0; + + code = tsdbReadBlockDataImpl(pReader, &pBlockL->bInfo, 1, aColId, nColId, pBlockData); + if (code) goto _err; + + return code; + +_err: + tsdbError("vgId:%d tsdb read last block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + return code; +} + #if 0 static int32_t tsdbReadBlockDataKey(SBlockData *pBlockData, SBlockInfo *pSubBlock, uint8_t *pBuf, uint8_t **ppBuf) { int32_t code = 0; From 7671bfe19e03d87acfcc67be7dfcfad3056cef45 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 7 Aug 2022 15:20:28 +0000 Subject: [PATCH 27/92] more refact --- source/dnode/vnode/src/inc/tsdb.h | 1 + .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 81 ++++++++----------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 34 ++++++++ 3 files changed, 67 insertions(+), 49 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 6af1a1a79b..e1bbc3278d 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -188,6 +188,7 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol uint8_t **ppBuf); int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, int32_t nVal, SColData *pColData, uint8_t **ppBuf); +int32_t tsdbReadAndCheckFile(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size); // tsdbMemTable ============================================================================================== // SMemTable int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 4f02299c7e..94d957eac7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -401,6 +401,7 @@ struct SDataFReader { uint8_t *pBuf1; uint8_t *pBuf2; + uint8_t *pBuf3; }; int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) { @@ -485,6 +486,7 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { tFree((*ppReader)->pBuf1); tFree((*ppReader)->pBuf2); + tFree((*ppReader)->pBuf3); taosMemoryFree(*ppReader); _exit: @@ -729,94 +731,75 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo int32_t nColId, SBlockData *pBlockData) { int32_t code = 0; + // TODO tBlockDataReset(pBlockData); TdFilePtr pFD = fromLast ? pReader->pLastFD : pReader->pDataFD; - // seek - int64_t n = taosLSeekFile(pFD, pBlkInfo->offset, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // read - code = tRealloc(&pReader->pBuf1, pBlkInfo->szBlock); + // uid + version + tskey + code = tsdbReadAndCheckFile(pFD, pBlkInfo->offset, &pReader->pBuf1, pBlkInfo->szKey); if (code) goto _err; - - n = taosReadFile(pFD, pReader->pBuf1, pBlkInfo->szBlock); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < pBlkInfo->szBlock) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - uint8_t *p = pReader->pBuf1; - // check & decode SDiskDataHdr hdr; - if (!taosCheckChecksumWhole(p, pBlkInfo->szKey)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - p += tGetDiskDataHdr(p, &hdr); + uint8_t *p = pReader->pBuf1 + tGetDiskDataHdr(pReader->pBuf1, &hdr); - tBlockDataSetSchema(pBlockData, NULL, hdr.suid, hdr.uid); + ASSERT(hdr.delimiter == TSDB_FILE_DLMT); + ASSERT(hdr.suid || hdr.uid); + + pBlockData->suid = hdr.suid; + pBlockData->uid = hdr.uid; pBlockData->nRow = hdr.nRow; + // uid if (hdr.uid == 0) { ASSERT(hdr.szUid); code = tsdbDecmprData(p, hdr.szUid, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aUid, sizeof(int64_t) * hdr.nRow, &pReader->pBuf2); if (code) goto _err; } else { - ASSERT(hdr.szUid == 0); + ASSERT(!hdr.szUid); } p += hdr.szUid; + // version code = tsdbDecmprData(p, hdr.szVer, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aVersion, sizeof(int64_t) * hdr.nRow, &pReader->pBuf2); if (code) goto _err; p += hdr.szVer; - code = tsdbDecmprData(p, hdr.szKey, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aTSKEY, + // TSKEY + code = tsdbDecmprData(p, hdr.szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr.cmprAlg, (uint8_t **)&pBlockData->aTSKEY, sizeof(TSKEY) * hdr.nRow, &pReader->pBuf2); if (code) goto _err; p += hdr.szKey; - p += sizeof(TSCKSUM); - // SBlockCol + ASSERT(p - pReader->pBuf1 == sizeof(TSCKSUM)); + + // read and decode columns if (hdr.szBlkCol > 0) { - if (!taosCheckChecksumWhole(p, hdr.szBlkCol + sizeof(TSCKSUM))) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } + code = + tsdbReadAndCheckFile(pFD, pBlkInfo->offset + pBlkInfo->szKey, &pReader->pBuf1, hdr.szBlkCol + sizeof(TSCKSUM)); + if (code) goto _err; - int32_t iColData = 0; - uint8_t *pt = p + hdr.szBlkCol + sizeof(TSCKSUM); - n = 0; + int32_t n = 0; while (n < hdr.szBlkCol) { SBlockCol blockCol; - n += tGetBlockCol(p + n, &blockCol); + n += tGetBlockCol(pReader->pBuf1 + n, &blockCol); ASSERT(blockCol.flag && blockCol.flag != HAS_NONE); - SColData *pColData; - code = tBlockDataAddColData(pBlockData, iColData, &pColData); - if (code) goto _err; - iColData++; + // TODO: merge with the column IDs - tColDataInit(pColData, blockCol.cid, blockCol.type, blockCol.smaOn); + SColData *pColData = NULL; // (todo) if (blockCol.flag == HAS_NULL) { - for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { - code = tColDataAppendValue(pColData, &COL_VAL_NULL(blockCol.cid, blockCol.type)); - if (code) goto _err; - } + // TODO: make a hdr.nRow COL_VAL_NULL(); } else { - code = tsdbDecmprColData(pt + blockCol.offset, &blockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->pBuf2); + code = tsdbReadAndCheckFile( + pFD, pBlkInfo->offset + pBlkInfo->szKey + hdr.szBlkCol + sizeof(TSCKSUM) + blockCol.offset, &pReader->pBuf2, + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM)); + + code = tsdbDecmprColData(pReader->pBuf2, &blockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->pBuf3); if (code) goto _err; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index dbb35532e8..6a27b10cf1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1865,6 +1865,40 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in } p += pBlockCol->szValue; +_exit: + return code; +} + +int32_t tsdbReadAndCheckFile(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size) { + int32_t code = 0; + + // alloc + code = tRealloc(ppOut, size); + if (code) goto _exit; + + // seek + int64_t n = taosLSeekFile(pFD, offset, SEEK_SET); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; + } + + // read + n = taosReadFile(pFD, *ppOut, size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; + } else if (n < size) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } + + // check + if (!taosCheckChecksumWhole(*ppOut, size)) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } + _exit: return code; } \ No newline at end of file From 33c3f34f3489732a57ce92c7a26abe3c6a5f7e55 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 Aug 2022 03:22:24 +0000 Subject: [PATCH 28/92] more work --- source/dnode/vnode/src/inc/tsdb.h | 17 +- source/dnode/vnode/src/tsdb/tsdbCache.c | 14 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 16 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 6 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 167 ++---------------- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 14 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 4 +- 7 files changed, 49 insertions(+), 189 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index e1bbc3278d..f33b8cd51d 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -137,8 +137,10 @@ int32_t tGetColData(uint8_t *p, SColData *pColData); #define tBlockDataLastRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1) #define tBlockDataFirstKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataFirstRow(PBLOCKDATA)) #define tBlockDataLastKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA)) -int32_t tBlockDataInit(SBlockData *pBlockData); -void tBlockDataClear(SBlockData *pBlockData, int8_t deepClear); + +int32_t tBlockDataCreate(SBlockData *pBlockData); +void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear); + void tBlockDataReset(SBlockData *pBlockData); int32_t tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema, int64_t suid, int64_t uid); int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); @@ -262,17 +264,6 @@ int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBl int32_t nColId); int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, int16_t *aColId, int32_t nColId); - -#if 0 -int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int16_t *aColId, int32_t nCol, - SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); -int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, SBlockData *pBlockData, - uint8_t **ppBuf1, uint8_t **ppBuf2); -int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, uint8_t **ppBuf1, - uint8_t **ppBuf2); -int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, uint8_t **ppBuf1, - uint8_t **ppBuf2); -#endif // SDelFWriter int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index c2bfad264b..2cb3f1f46d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -450,9 +450,9 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { if (--state->iFileSet >= 0) { pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet); } else { - // tBlockDataClear(&state->blockData, 1); + // tBlockDataDestroy(&state->blockData, 1); if (state->pBlockData) { - tBlockDataClear(state->pBlockData, 1); + tBlockDataDestroy(state->pBlockData, 1); state->pBlockData = NULL; } @@ -494,7 +494,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { if (!state->pBlockData) { state->pBlockData = &state->blockData; - tBlockDataInit(&state->blockData); + tBlockDataCreate(&state->blockData); } } case SFSNEXTROW_BLOCKDATA: @@ -552,8 +552,8 @@ _err: state->aBlockIdx = NULL; } if (state->pBlockData) { - // tBlockDataClear(&state->blockData, 1); - tBlockDataClear(state->pBlockData, 1); + // tBlockDataDestroy(&state->blockData, 1); + tBlockDataDestroy(state->pBlockData, 1); state->pBlockData = NULL; } @@ -579,8 +579,8 @@ int32_t clearNextRowFromFS(void *iter) { state->aBlockIdx = NULL; } if (state->pBlockData) { - // tBlockDataClear(&state->blockData, 1); - tBlockDataClear(state->pBlockData, 1); + // tBlockDataDestroy(&state->blockData, 1); + tBlockDataDestroy(state->pBlockData, 1); state->pBlockData = NULL; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index ad721ede7a..44f360d597 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1270,7 +1270,7 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { goto _exit; } - code = tBlockDataInit(&pCommitter->dReader.bData); + code = tBlockDataCreate(&pCommitter->dReader.bData); if (code) goto _exit; pCommitter->dReader.aBlockL = taosArrayInit(0, sizeof(SBlockL)); @@ -1279,7 +1279,7 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { goto _exit; } - code = tBlockDataInit(&pCommitter->dReader.bDatal); + code = tBlockDataCreate(&pCommitter->dReader.bDatal); if (code) goto _exit; // Writer @@ -1295,10 +1295,10 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { goto _exit; } - code = tBlockDataInit(&pCommitter->dWriter.bData); + code = tBlockDataCreate(&pCommitter->dWriter.bData); if (code) goto _exit; - code = tBlockDataInit(&pCommitter->dWriter.bDatal); + code = tBlockDataCreate(&pCommitter->dWriter.bDatal); if (code) goto _exit; _exit: @@ -1309,16 +1309,16 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { // Reader taosArrayDestroy(pCommitter->dReader.aBlockIdx); tMapDataClear(&pCommitter->dReader.mBlock); - tBlockDataClear(&pCommitter->dReader.bData, 1); + tBlockDataDestroy(&pCommitter->dReader.bData, 1); taosArrayDestroy(pCommitter->dReader.aBlockL); - tBlockDataClear(&pCommitter->dReader.bDatal, 1); + tBlockDataDestroy(&pCommitter->dReader.bDatal, 1); // Writer taosArrayDestroy(pCommitter->dWriter.aBlockIdx); taosArrayDestroy(pCommitter->dWriter.aBlockL); tMapDataClear(&pCommitter->dWriter.mBlock); - tBlockDataClear(&pCommitter->dWriter.bData, 1); - tBlockDataClear(&pCommitter->dWriter.bDatal, 1); + tBlockDataDestroy(&pCommitter->dWriter.bData, 1); + tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1); tTSchemaDestroy(pCommitter->skmTable.pTSchema); tTSchemaDestroy(pCommitter->skmRow.pTSchema); } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 0157f0ed22..cd1e46c342 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -438,7 +438,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pSup->tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; - code = tBlockDataInit(&pReader->status.fileBlockData); + code = tBlockDataCreate(&pReader->status.fileBlockData); if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _end; @@ -2670,7 +2670,7 @@ void tsdbReaderClose(STsdbReader* pReader) { } } taosMemoryFree(pSupInfo->buildBuf); - tBlockDataClear(&pReader->status.fileBlockData, true); + tBlockDataDestroy(&pReader->status.fileBlockData, true); cleanupDataBlockIterator(&pReader->status.blockIter); @@ -2874,7 +2874,7 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) { tBlockDataClearData(&pStatus->fileBlockData); int32_t code = doLoadFileBlockData(pReader, &pStatus->blockIter, pBlockScanInfo, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { - tBlockDataClear(&pStatus->fileBlockData, 1); + tBlockDataDestroy(&pStatus->fileBlockData, 1); terrno = code; return NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 94d957eac7..69f39bb3c5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -846,63 +846,6 @@ _err: } #if 0 -static int32_t tsdbReadBlockDataKey(SBlockData *pBlockData, SBlockInfo *pSubBlock, uint8_t *pBuf, uint8_t **ppBuf) { - int32_t code = 0; -#if 0 - int64_t size = pSubBlock->szVersion + pSubBlock->szTSKEY + sizeof(TSCKSUM); - int64_t n; - - if (!taosCheckChecksumWhole(pBuf, size)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - code = tRealloc((uint8_t **)&pBlockData->aVersion, sizeof(int64_t) * pSubBlock->nRow); - if (code) goto _err; - code = tRealloc((uint8_t **)&pBlockData->aTSKEY, sizeof(TSKEY) * pSubBlock->nRow); - if (code) goto _err; - - if (pSubBlock->cmprAlg == NO_COMPRESSION) { - ASSERT(pSubBlock->szVersion == sizeof(int64_t) * pSubBlock->nRow); - ASSERT(pSubBlock->szTSKEY == sizeof(TSKEY) * pSubBlock->nRow); - - // VERSION - memcpy(pBlockData->aVersion, pBuf, pSubBlock->szVersion); - - // TSKEY - memcpy(pBlockData->aTSKEY, pBuf + pSubBlock->szVersion, pSubBlock->szTSKEY); - } else { - size = sizeof(int64_t) * pSubBlock->nRow + COMP_OVERFLOW_BYTES; - if (pSubBlock->cmprAlg == TWO_STAGE_COMP) { - code = tRealloc(ppBuf, size); - if (code) goto _err; - } - - // VERSION - n = tsDecompressBigint(pBuf, pSubBlock->szVersion, pSubBlock->nRow, (char *)pBlockData->aVersion, - sizeof(int64_t) * pSubBlock->nRow, pSubBlock->cmprAlg, *ppBuf, size); - if (n < 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _err; - } - - // TSKEY - n = tsDecompressTimestamp(pBuf + pSubBlock->szVersion, pSubBlock->szTSKEY, pSubBlock->nRow, - (char *)pBlockData->aTSKEY, sizeof(TSKEY) * pSubBlock->nRow, pSubBlock->cmprAlg, *ppBuf, - size); - if (n < 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _err; - } - } - - return code; - -_err: -#endif - return code; -} - static int32_t tsdbReadColDataImpl(SBlockInfo *pSubBlock, SBlockCol *pBlockCol, SColData *pColData, uint8_t *pBuf, uint8_t **ppBuf) { int32_t code = 0; @@ -998,80 +941,6 @@ _err: return code; } -#if 0 -static int32_t tsdbReadBlockCol(uint8_t *pBuf, int32_t szBlockCol, SDiskDataHdr *pHdr, SArray *aBlockCol) { - int32_t code = 0; - int32_t n = 0; - SBlockCol blockCol; - SBlockCol *pBlockCol = &blockCol; - - // checksum - if (!taosCheckChecksumWhole(pBuf, szBlockCol + sizeof(TSCKSUM))) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // hdr - *pHdr = *(SDiskDataHdr *)pBuf; - n += sizeof(SDiskDataHdr); - - // aBlockCol - while (n < szBlockCol) { - n += tGetBlockCol(pBuf + n, pBlockCol); - - if (taosArrayPush(aBlockCol, pBlockCol) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - } - - ASSERT(n == szBlockCol); - - return code; - -_err: - return code; -} -#endif - -static int32_t tsdbReadDataArray(uint8_t *pInput, int32_t szInput, int32_t nEle, int8_t type, int8_t cmprAlg, - uint8_t **ppOut, uint8_t **ppBuf) { - int32_t code = 0; - int32_t size; - - // size - if (IS_VAR_DATA_TYPE(type)) { - size = nEle; - } else { - size = tDataTypes[type].bytes * nEle; - } - - // alloc - code = tRealloc(ppOut, size); - if (code) goto _exit; - - // decode - if (cmprAlg == NO_COMPRESSION) { - ASSERT(szInput == size); - memcpy(*ppOut, pInput, size); - } else { - if (cmprAlg == TWO_STAGE_COMP) { - code = tRealloc(ppBuf, size + COMP_OVERFLOW_BYTES); - if (code) goto _exit; - - int32_t n = - tDataTypes[type].decompFunc(pInput, szInput, nEle, *ppOut, size, cmprAlg, *ppBuf, size + COMP_OVERFLOW_BYTES); - if (n <= 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _exit; - } - } - } - -_exit: - return code; -} - static int32_t tsdbReadSubColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int32_t iSubBlock, int16_t *aColId, int32_t nCol, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2) { @@ -1210,29 +1079,29 @@ int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBl SBlockData *pBlockData1 = &(SBlockData){0}; SBlockData *pBlockData2 = &(SBlockData){0}; - tBlockDataInit(pBlockData1); - tBlockDataInit(pBlockData2); + tBlockDataCreate(pBlockData1); + tBlockDataCreate(pBlockData2); for (int32_t iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { code = tsdbReadSubColData(pReader, pBlockIdx, pBlock, iSubBlock, aColId, nCol, pBlockData1, ppBuf1, ppBuf2); if (code) goto _err; code = tBlockDataCopy(pBlockData, pBlockData2); if (code) { - tBlockDataClear(pBlockData1, 1); - tBlockDataClear(pBlockData2, 1); + tBlockDataDestroy(pBlockData1, 1); + tBlockDataDestroy(pBlockData2, 1); goto _err; } code = tBlockDataMerge(pBlockData1, pBlockData2, pBlockData); if (code) { - tBlockDataClear(pBlockData1, 1); - tBlockDataClear(pBlockData2, 1); + tBlockDataDestroy(pBlockData1, 1); + tBlockDataDestroy(pBlockData2, 1); goto _err; } } - tBlockDataClear(pBlockData1, 1); - tBlockDataClear(pBlockData2, 1); + tBlockDataDestroy(pBlockData1, 1); + tBlockDataDestroy(pBlockData2, 1); } tFree(pBuf1); @@ -1349,34 +1218,34 @@ int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBl SBlockData *pBlockData1 = &(SBlockData){0}; SBlockData *pBlockData2 = &(SBlockData){0}; - tBlockDataInit(pBlockData1); - tBlockDataInit(pBlockData2); + tBlockDataCreate(pBlockData1); + tBlockDataCreate(pBlockData2); for (iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { code = tsdbReadSubBlockData(pReader, pBlock, iSubBlock, pBlockData1, ppBuf1, ppBuf2); if (code) { - tBlockDataClear(pBlockData1, 1); - tBlockDataClear(pBlockData2, 1); + tBlockDataDestroy(pBlockData1, 1); + tBlockDataDestroy(pBlockData2, 1); goto _err; } code = tBlockDataCopy(pBlockData, pBlockData2); if (code) { - tBlockDataClear(pBlockData1, 1); - tBlockDataClear(pBlockData2, 1); + tBlockDataDestroy(pBlockData1, 1); + tBlockDataDestroy(pBlockData2, 1); goto _err; } // merge two block data code = tBlockDataMerge(pBlockData1, pBlockData2, pBlockData); if (code) { - tBlockDataClear(pBlockData1, 1); - tBlockDataClear(pBlockData2, 1); + tBlockDataDestroy(pBlockData1, 1); + tBlockDataDestroy(pBlockData2, 1); goto _err; } } - tBlockDataClear(pBlockData1, 1); - tBlockDataClear(pBlockData2, 1); + tBlockDataDestroy(pBlockData1, 1); + tBlockDataDestroy(pBlockData2, 1); } ASSERT(pBlock->nRow == pBlockData->nRow); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 06a38303ec..973cd1e53a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -289,9 +289,9 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type goto _err; } pReader->mBlock = tMapDataInit(); - code = tBlockDataInit(&pReader->oBlockData); + code = tBlockDataCreate(&pReader->oBlockData); if (code) goto _err; - code = tBlockDataInit(&pReader->nBlockData); + code = tBlockDataCreate(&pReader->nBlockData); if (code) goto _err; pReader->aDelIdx = taosArrayInit(0, sizeof(SDelIdx)); @@ -327,8 +327,8 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { } taosArrayDestroy(pReader->aBlockIdx); tMapDataClear(&pReader->mBlock); - tBlockDataClear(&pReader->oBlockData, 1); - tBlockDataClear(&pReader->nBlockData, 1); + tBlockDataDestroy(&pReader->oBlockData, 1); + tBlockDataDestroy(&pReader->nBlockData, 1); if (pReader->pDelFReader) { tsdbDelFReaderClose(&pReader->pDelFReader); @@ -1123,7 +1123,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr pWriter->commitID = pTsdb->pVnode->state.commitID; // for data file - code = tBlockDataInit(&pWriter->bData); + code = tBlockDataCreate(&pWriter->bData); if (code) goto _err; pWriter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); @@ -1131,7 +1131,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - code = tBlockDataInit(&pWriter->bDataR); + code = tBlockDataCreate(&pWriter->bDataR); if (code) goto _err; pWriter->aBlockIdxW = taosArrayInit(0, sizeof(SBlockIdx)); @@ -1139,7 +1139,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - code = tBlockDataInit(&pWriter->bDataW); + code = tBlockDataCreate(&pWriter->bDataW); if (code) goto _err; // for del file diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 6a27b10cf1..039fe54f3d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1107,7 +1107,7 @@ static FORCE_INLINE int32_t tColDataCmprFn(const void *p1, const void *p2) { } // SBlockData ====================================================== -int32_t tBlockDataInit(SBlockData *pBlockData) { +int32_t tBlockDataCreate(SBlockData *pBlockData) { int32_t code = 0; pBlockData->suid = 0; @@ -1132,7 +1132,7 @@ _exit: return code; } -void tBlockDataClear(SBlockData *pBlockData, int8_t deepClear) { +void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear) { tFree((uint8_t *)pBlockData->aUid); tFree((uint8_t *)pBlockData->aVersion); tFree((uint8_t *)pBlockData->aTSKEY); From 8ae2ab1c6e92d044920aa9fed3506acc9e416352 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 Aug 2022 05:34:01 +0000 Subject: [PATCH 29/92] more work --- source/dnode/vnode/src/inc/tsdb.h | 31 ++++----- source/dnode/vnode/src/tsdb/tsdbCommit.c | 10 +-- source/dnode/vnode/src/tsdb/tsdbRead.c | 6 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 13 ++-- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 6 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 66 ++++++++++++++----- 6 files changed, 82 insertions(+), 50 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index f33b8cd51d..0cbbe4c351 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -138,22 +138,23 @@ int32_t tGetColData(uint8_t *p, SColData *pColData); #define tBlockDataFirstKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataFirstRow(PBLOCKDATA)) #define tBlockDataLastKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA)) -int32_t tBlockDataCreate(SBlockData *pBlockData); -void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear); - -void tBlockDataReset(SBlockData *pBlockData); -int32_t tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema, int64_t suid, int64_t uid); -int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); -int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); -void tBlockDataClearData(SBlockData *pBlockData); - -int32_t tBlockDataCorrectSchema(SBlockData *pBlockData, SBlockData *pBlockDataFrom); -int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData); -int32_t tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest); +int32_t tBlockDataCreate(SBlockData *pBlockData); +void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear); +int32_t tBlockDataInit(SBlockData *pBlockData, int64_t suid, int64_t uid, STSchema *pTSchema); +int32_t tBlockDataInitEx(SBlockData *pBlockData, int64_t *suid, int64_t uid, SArray *aColId); +void tBlockDataReset(SBlockData *pBlockData); +int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); +void tBlockDataClear(SBlockData *pBlockData); SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx); void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData); -int32_t tPutBlockData(uint8_t *p, SBlockData *pBlockData); -int32_t tGetBlockData(uint8_t *p, SBlockData *pBlockData); + +#if 1 +int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); +int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData); +int32_t tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest); +int32_t tPutBlockData(uint8_t *p, SBlockData *pBlockData); +int32_t tGetBlockData(uint8_t *p, SBlockData *pBlockData); +#endif // SDiskDataHdr int32_t tPutDiskDataHdr(uint8_t *p, void *ph); int32_t tGetDiskDataHdr(uint8_t *p, void *ph); @@ -190,7 +191,7 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol uint8_t **ppBuf); int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, int32_t nVal, SColData *pColData, uint8_t **ppBuf); -int32_t tsdbReadAndCheckFile(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size); +int32_t tsdbReadAndCheckFile(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck); // tsdbMemTable ============================================================================================== // SMemTable int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 44f360d597..d48125beb5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -539,7 +539,7 @@ static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) { if (code) goto _err; // clear - tBlockDataClearData(pBlockData); + tBlockDataClear(pBlockData); return code; @@ -578,7 +578,7 @@ static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { } // clear - tBlockDataClearData(pBlockData); + tBlockDataClear(pBlockData); return code; @@ -596,7 +596,7 @@ static int32_t tsdbMergeCommitData(SCommitter *pCommitter, STbDataIter *pIter, S code = tsdbReadDataBlock(pCommitter->dReader.pReader, pBlock, pBlockDataR, NULL, 0); if (code) goto _err; - tBlockDataClearData(pBlockDataW); + tBlockDataClear(pBlockDataW); int32_t iRow = 0; TSDBROW row; TSDBROW *pRow1 = tsdbTbDataIterGet(pIter); @@ -672,7 +672,7 @@ static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter STbData *pTbData = pIter->pTbData; SBlockData *pBlockData = &pCommitter->dWriter.bData; - tBlockDataClearData(pBlockData); + tBlockDataClear(pBlockData); TSDBROW *pRow = tsdbTbDataIterGet(pIter); while (true) { if (pRow == NULL) { @@ -741,7 +741,7 @@ static int32_t tsdbMergeAsSubBlock(SCommitter *pCommitter, STbDataIter *pIter, S STbData *pTbData = pIter->pTbData; SBlockData *pBlockData = &pCommitter->dWriter.bData; - tBlockDataClearData(pBlockData); + tBlockDataClear(pBlockData); TSDBROW *pRow = tsdbTbDataIterGet(pIter); while (true) { if (pRow == NULL) break; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index cd1e46c342..baafdb488c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1748,7 +1748,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { TSDBKEY key = getCurrentKeyInBuf(pBlockIter, pReader); if (fileBlockShouldLoad(pReader, pFBlock, pBlock, pScanInfo, key)) { tBlockDataReset(&pStatus->fileBlockData); - tBlockDataClearData(&pStatus->fileBlockData); + tBlockDataClear(&pStatus->fileBlockData); code = doLoadFileBlockData(pReader, pBlockIter, pScanInfo, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2208,7 +2208,7 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn // 3. load the neighbor block, and set it to be the currently accessed file data block tBlockDataReset(&pStatus->fileBlockData); - tBlockDataClearData(&pStatus->fileBlockData); + tBlockDataClear(&pStatus->fileBlockData); int32_t code = doLoadFileBlockData(pReader, pBlockIter, pScanInfo, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2871,7 +2871,7 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) { STableBlockScanInfo* pBlockScanInfo = taosHashGet(pStatus->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); tBlockDataReset(&pStatus->fileBlockData); - tBlockDataClearData(&pStatus->fileBlockData); + tBlockDataClear(&pStatus->fileBlockData); int32_t code = doLoadFileBlockData(pReader, &pStatus->blockIter, pBlockScanInfo, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { tBlockDataDestroy(&pStatus->fileBlockData, 1); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 69f39bb3c5..17ab9eff67 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -731,13 +731,14 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo int32_t nColId, SBlockData *pBlockData) { int32_t code = 0; - // TODO - tBlockDataReset(pBlockData); + ASSERT(pBlockData->suid || pBlockData->uid); + + tBlockDataClear(pBlockData); TdFilePtr pFD = fromLast ? pReader->pLastFD : pReader->pDataFD; // uid + version + tskey - code = tsdbReadAndCheckFile(pFD, pBlkInfo->offset, &pReader->pBuf1, pBlkInfo->szKey); + code = tsdbReadAndCheckFile(pFD, pBlkInfo->offset, &pReader->pBuf1, pBlkInfo->szKey, 1); if (code) goto _err; SDiskDataHdr hdr; uint8_t *p = pReader->pBuf1 + tGetDiskDataHdr(pReader->pBuf1, &hdr); @@ -776,8 +777,8 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo // read and decode columns if (hdr.szBlkCol > 0) { - code = - tsdbReadAndCheckFile(pFD, pBlkInfo->offset + pBlkInfo->szKey, &pReader->pBuf1, hdr.szBlkCol + sizeof(TSCKSUM)); + code = tsdbReadAndCheckFile(pFD, pBlkInfo->offset + pBlkInfo->szKey, &pReader->pBuf1, + hdr.szBlkCol + sizeof(TSCKSUM), 1); if (code) goto _err; int32_t n = 0; @@ -797,7 +798,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo } else { code = tsdbReadAndCheckFile( pFD, pBlkInfo->offset + pBlkInfo->szKey + hdr.szBlkCol + sizeof(TSCKSUM) + blockCol.offset, &pReader->pBuf2, - blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM)); + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM), 1); code = tsdbDecmprColData(pReader->pBuf2, &blockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->pBuf3); if (code) goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 973cd1e53a..4064ad950c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -482,7 +482,7 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { if (code) goto _err; tBlockReset(&pWriter->blockW); - tBlockDataClearData(&pWriter->bDataW); + tBlockDataClear(&pWriter->bDataW); } pWriter->iRow++; @@ -675,7 +675,7 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { if (code) goto _err; tBlockReset(&pWriter->blockW); - tBlockDataClearData(&pWriter->bDataW); + tBlockDataClear(&pWriter->bDataW); } code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); @@ -725,7 +725,7 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { if (code) goto _err; tBlockReset(&pWriter->blockW); - tBlockDataClearData(&pWriter->bDataW); + tBlockDataClear(&pWriter->bDataW); } return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 039fe54f3d..92495096da 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1145,31 +1145,54 @@ void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear) { pBlockData->aColData = NULL; } -void tBlockDataReset(SBlockData *pBlockData) { - pBlockData->suid = 0; - pBlockData->uid = 0; - pBlockData->nRow = 0; - taosArrayClear(pBlockData->aIdx); -} - -int32_t tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema, int64_t suid, int64_t uid) { +int32_t tBlockDataInit(SBlockData *pBlockData, int64_t suid, int64_t uid, STSchema *pTSchema) { int32_t code = 0; ASSERT(suid || uid); - tBlockDataReset(pBlockData); pBlockData->suid = suid; pBlockData->uid = uid; + pBlockData->nRow = 0; + + taosArrayClear(pBlockData->aIdx); + for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { + STColumn *pTColumn = &pTSchema->columns[iColumn]; + + SColData *pColData; + code = tBlockDataAddColData(pBlockData, iColumn - 1, &pColData); + if (code) goto _exit; + + tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) ? 1 : 0); + } + +_exit: + return code; +} + +int32_t tBlockDataInitEx(SBlockData *pBlockData, int64_t *suid, int64_t uid, SArray *aColId) { + int32_t code = 0; + + ASSERT(suid || uid); + + pBlockData->suid = suid; + pBlockData->uid = uid; + pBlockData->nRow = 0; + + taosArrayClear(pBlockData->aIdx); + if (aColId) { + int16_t lcid = -1; + for (int32_t iColId = 0; iColId < taosArrayGetSize(aColId); iColId++) { + int16_t cid = *(int16_t *)taosArrayGet(aColId, iColId); + + ASSERT(cid != PRIMARYKEY_TIMESTAMP_COL_ID); + ASSERT(cid > lcid); + lcid = cid; - if (pTSchema) { - for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { - STColumn *pTColumn = &pTSchema->columns[iColumn]; SColData *pColData; - - code = tBlockDataAddColData(pBlockData, iColumn - 1, &pColData); + code = tBlockDataAddColData(pBlockData, iColId, &pColData); if (code) goto _exit; - tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) != 0); + tColDataInit(pColData, cid, TSDB_DATA_TYPE_NULL, -1); } } @@ -1177,7 +1200,14 @@ _exit: return code; } -void tBlockDataClearData(SBlockData *pBlockData) { +void tBlockDataReset(SBlockData *pBlockData) { + pBlockData->suid = 0; + pBlockData->uid = 0; + pBlockData->nRow = 0; + taosArrayClear(pBlockData->aIdx); +} + +void tBlockDataClear(SBlockData *pBlockData) { pBlockData->nRow = 0; for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); @@ -1869,7 +1899,7 @@ _exit: return code; } -int32_t tsdbReadAndCheckFile(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size) { +int32_t tsdbReadAndCheckFile(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck) { int32_t code = 0; // alloc @@ -1894,7 +1924,7 @@ int32_t tsdbReadAndCheckFile(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int } // check - if (!taosCheckChecksumWhole(*ppOut, size)) { + if (toCheck && !taosCheckChecksumWhole(*ppOut, size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _exit; } From 6baae7d50fd70707382d24de0f3fea78d1425978 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 Aug 2022 06:43:51 +0000 Subject: [PATCH 30/92] more work --- source/dnode/vnode/src/inc/tsdb.h | 2 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 78 ++++++++++++------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 9 ++- 3 files changed, 57 insertions(+), 32 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 0cbbe4c351..2606d7a8d0 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -191,7 +191,7 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol uint8_t **ppBuf); int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, int32_t nVal, SColData *pColData, uint8_t **ppBuf); -int32_t tsdbReadAndCheckFile(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck); +int32_t tsdbReadAndCheck(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck); // tsdbMemTable ============================================================================================== // SMemTable int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 17ab9eff67..48023858cc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -727,27 +727,24 @@ _err: return code; } -static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo, int8_t fromLast, int16_t *aColId, - int32_t nColId, SBlockData *pBlockData) { +static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo, int8_t fromLast, + SBlockData *pBlockData) { int32_t code = 0; - ASSERT(pBlockData->suid || pBlockData->uid); - tBlockDataClear(pBlockData); TdFilePtr pFD = fromLast ? pReader->pLastFD : pReader->pDataFD; // uid + version + tskey - code = tsdbReadAndCheckFile(pFD, pBlkInfo->offset, &pReader->pBuf1, pBlkInfo->szKey, 1); + code = tsdbReadAndCheck(pFD, pBlkInfo->offset, &pReader->pBuf1, pBlkInfo->szKey, 1); if (code) goto _err; SDiskDataHdr hdr; uint8_t *p = pReader->pBuf1 + tGetDiskDataHdr(pReader->pBuf1, &hdr); ASSERT(hdr.delimiter == TSDB_FILE_DLMT); - ASSERT(hdr.suid || hdr.uid); + ASSERT(pBlockData->suid == hdr.suid); + ASSERT(pBlockData->uid == hdr.uid); - pBlockData->suid = hdr.suid; - pBlockData->uid = hdr.uid; pBlockData->nRow = hdr.nRow; // uid @@ -776,36 +773,61 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo ASSERT(p - pReader->pBuf1 == sizeof(TSCKSUM)); // read and decode columns + if (taosArrayGetSize(pBlockData->aIdx) == 0) goto _exit; + if (hdr.szBlkCol > 0) { - code = tsdbReadAndCheckFile(pFD, pBlkInfo->offset + pBlkInfo->szKey, &pReader->pBuf1, - hdr.szBlkCol + sizeof(TSCKSUM), 1); + int64_t offset = pBlkInfo->offset + pBlkInfo->szKey; + code = tsdbReadAndCheck(pFD, offset, &pReader->pBuf1, hdr.szBlkCol + sizeof(TSCKSUM), 1); if (code) goto _err; + } - int32_t n = 0; - while (n < hdr.szBlkCol) { - SBlockCol blockCol; + SBlockCol blockCol = {.cid = 0}; + SBlockCol *pBlockCol = &blockCol; + int32_t n = 0; - n += tGetBlockCol(pReader->pBuf1 + n, &blockCol); + for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { + SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - ASSERT(blockCol.flag && blockCol.flag != HAS_NONE); - - // TODO: merge with the column IDs - - SColData *pColData = NULL; // (todo) - - if (blockCol.flag == HAS_NULL) { - // TODO: make a hdr.nRow COL_VAL_NULL(); + while (pBlockCol && pBlockCol->cid < pColData->cid) { + if (n < hdr.szBlkCol) { + n += tGetBlockCol(pReader->pBuf1 + n, pBlockCol); } else { - code = tsdbReadAndCheckFile( - pFD, pBlkInfo->offset + pBlkInfo->szKey + hdr.szBlkCol + sizeof(TSCKSUM) + blockCol.offset, &pReader->pBuf2, - blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM), 1); + ASSERT(n == hdr.szBlkCol); + pBlockCol = NULL; + } + } - code = tsdbDecmprColData(pReader->pBuf2, &blockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->pBuf3); + if (pBlockCol == NULL || pBlockCol->cid > pColData->cid) { + // add a lot of NONE + for (int32_t iRow = 0; iRow < hdr.nRow; iRow++) { + code = tColDataAppendValue(pColData, &COL_VAL_NONE(pBlockCol->cid, pBlockCol->type)); + if (code) goto _err; + } + } else { + ASSERT(pBlockCol->type == pColData->type); + ASSERT(pBlockCol->flag && pBlockCol->flag != HAS_NONE); + + if (pBlockCol->flag == HAS_NULL) { + // add a lot of NULL + for (int32_t iRow = 0; iRow < hdr.nRow; iRow++) { + code = tColDataAppendValue(pColData, &COL_VAL_NULL(pBlockCol->cid, pBlockCol->type)); + if (code) goto _err; + } + } else { + // decode from binary + int64_t offset = pBlkInfo->offset + pBlkInfo->szKey + hdr.szBlkCol + sizeof(TSCKSUM) + pBlockCol->offset; + int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM); + + code = tsdbReadAndCheck(pFD, offset, &pReader->pBuf2, size, 0); + if (code) goto _err; + + code = tsdbDecmprColData(pReader->pBuf2, pBlockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->pBuf3); if (code) goto _err; } } } +_exit: return code; _err: @@ -817,7 +839,7 @@ int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBl int32_t nColId) { int32_t code = 0; - code = tsdbReadBlockDataImpl(pReader, &pBlock->aSubBlock[0], 0, aColId, nColId, pBlockData); + code = tsdbReadBlockDataImpl(pReader, &pBlock->aSubBlock[0], 0, pBlockData); if (code) goto _err; for (int32_t iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { @@ -836,7 +858,7 @@ int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *p int32_t nColId) { int32_t code = 0; - code = tsdbReadBlockDataImpl(pReader, &pBlockL->bInfo, 1, aColId, nColId, pBlockData); + code = tsdbReadBlockDataImpl(pReader, &pBlockL->bInfo, 1, pBlockData); if (code) goto _err; return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 92495096da..e4ebc3e839 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1172,6 +1172,7 @@ _exit: int32_t tBlockDataInitEx(SBlockData *pBlockData, int64_t *suid, int64_t uid, SArray *aColId) { int32_t code = 0; + ASSERT(0); ASSERT(suid || uid); pBlockData->suid = suid; @@ -1208,6 +1209,8 @@ void tBlockDataReset(SBlockData *pBlockData) { } void tBlockDataClear(SBlockData *pBlockData) { + ASSERT(pBlockData->suid || pBlockData->uid); + pBlockData->nRow = 0; for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); @@ -1864,8 +1867,8 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in goto _exit; } - pColData->cid = pBlockCol->cid; - pColData->type = pBlockCol->type; + ASSERT(pColData->cid == pBlockCol->cid); + ASSERT(pColData->type == pBlockCol->type); pColData->smaOn = pBlockCol->smaOn; pColData->flag = pBlockCol->flag; pColData->nVal = nVal; @@ -1899,7 +1902,7 @@ _exit: return code; } -int32_t tsdbReadAndCheckFile(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck) { +int32_t tsdbReadAndCheck(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck) { int32_t code = 0; // alloc From 49fde0b1a87937a3c6c94551c51f2ad94d6c95af Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 Aug 2022 07:27:44 +0000 Subject: [PATCH 31/92] more work --- source/dnode/vnode/src/inc/tsdb.h | 12 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 593 ++---------------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 206 +++--- 3 files changed, 139 insertions(+), 672 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 2606d7a8d0..0e10d1b000 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -141,17 +141,17 @@ int32_t tGetColData(uint8_t *p, SColData *pColData); int32_t tBlockDataCreate(SBlockData *pBlockData); void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear); int32_t tBlockDataInit(SBlockData *pBlockData, int64_t suid, int64_t uid, STSchema *pTSchema); -int32_t tBlockDataInitEx(SBlockData *pBlockData, int64_t *suid, int64_t uid, SArray *aColId); +int32_t tBlockDataInitEx(SBlockData *pBlockData, SBlockData *pBlockDataFrom); void tBlockDataReset(SBlockData *pBlockData); int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); void tBlockDataClear(SBlockData *pBlockData); SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx); void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData); +int32_t tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest); +int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData); #if 1 int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); -int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData); -int32_t tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest); int32_t tPutBlockData(uint8_t *p, SBlockData *pBlockData); int32_t tGetBlockData(uint8_t *p, SBlockData *pBlockData); #endif @@ -261,10 +261,8 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx); int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData); int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL); int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg); -int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, int16_t *aColId, - int32_t nColId); -int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, int16_t *aColId, - int32_t nColId); +int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData); +int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData); // SDelFWriter int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 48023858cc..f73883d050 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -835,16 +835,51 @@ _err: return code; } -int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, int16_t *aColId, - int32_t nColId) { +int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData) { int32_t code = 0; code = tsdbReadBlockDataImpl(pReader, &pBlock->aSubBlock[0], 0, pBlockData); if (code) goto _err; - for (int32_t iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { - // TODO - ASSERT(0); + if (pBlock->nSubBlock > 1) { + SBlockData bData1; + SBlockData bData2; + + // create + code = tBlockDataCreate(&bData1); + if (code) goto _err; + code = tBlockDataCreate(&bData2); + if (code) goto _err; + + // init + tBlockDataInitEx(&bData1, pBlockData); + tBlockDataInitEx(&bData2, pBlockData); + + for (int32_t iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { + code = tsdbReadBlockDataImpl(pReader, &pBlock->aSubBlock[iSubBlock], 0, &bData1); + if (code) { + tBlockDataDestroy(&bData1, 1); + tBlockDataDestroy(&bData2, 1); + goto _err; + } + + code = tBlockDataCopy(pBlockData, &bData2); + if (code) { + tBlockDataDestroy(&bData1, 1); + tBlockDataDestroy(&bData2, 1); + goto _err; + } + + code = tBlockDataMerge(&bData1, &bData2, pBlockData); + if (code) { + tBlockDataDestroy(&bData1, 1); + tBlockDataDestroy(&bData2, 1); + goto _err; + } + } + + tBlockDataDestroy(&bData1, 1); + tBlockDataDestroy(&bData2, 1); } return code; @@ -854,8 +889,7 @@ _err: return code; } -int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, int16_t *aColId, - int32_t nColId) { +int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData) { int32_t code = 0; code = tsdbReadBlockDataImpl(pReader, &pBlockL->bInfo, 1, pBlockData); @@ -868,551 +902,6 @@ _err: return code; } -#if 0 -static int32_t tsdbReadColDataImpl(SBlockInfo *pSubBlock, SBlockCol *pBlockCol, SColData *pColData, uint8_t *pBuf, - uint8_t **ppBuf) { - int32_t code = 0; -#if 0 - int64_t size; - int64_t n; - - if (!taosCheckChecksumWhole(pBuf, pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM))) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - pColData->nVal = pSubBlock->nRow; - pColData->flag = pBlockCol->flag; - - // BITMAP - if (pBlockCol->flag != HAS_VALUE) { - ASSERT(pBlockCol->szBitmap); - - size = BIT2_SIZE(pColData->nVal); - code = tRealloc(&pColData->pBitMap, size); - if (code) goto _err; - - code = tRealloc(ppBuf, size + COMP_OVERFLOW_BYTES); - if (code) goto _err; - - n = tsDecompressTinyint(pBuf, pBlockCol->szBitmap, size, pColData->pBitMap, size, TWO_STAGE_COMP, *ppBuf, - size + COMP_OVERFLOW_BYTES); - if (n <= 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _err; - } - - ASSERT(n == size); - } else { - ASSERT(pBlockCol->szBitmap == 0); - } - pBuf = pBuf + pBlockCol->szBitmap; - - // OFFSET - if (IS_VAR_DATA_TYPE(pColData->type)) { - ASSERT(pBlockCol->szOffset); - - size = sizeof(int32_t) * pColData->nVal; - code = tRealloc((uint8_t **)&pColData->aOffset, size); - if (code) goto _err; - - code = tRealloc(ppBuf, size + COMP_OVERFLOW_BYTES); - if (code) goto _err; - - n = tsDecompressInt(pBuf, pBlockCol->szOffset, pColData->nVal, (char *)pColData->aOffset, size, TWO_STAGE_COMP, - *ppBuf, size + COMP_OVERFLOW_BYTES); - if (n <= 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _err; - } - - ASSERT(n == size); - } else { - ASSERT(pBlockCol->szOffset == 0); - } - pBuf = pBuf + pBlockCol->szOffset; - - // VALUE - pColData->nData = pBlockCol->szOrigin; - - code = tRealloc(&pColData->pData, pColData->nData); - if (code) goto _err; - - if (pSubBlock->cmprAlg == NO_COMPRESSION) { - memcpy(pColData->pData, pBuf, pColData->nData); - } else { - if (pSubBlock->cmprAlg == TWO_STAGE_COMP) { - code = tRealloc(ppBuf, pColData->nData + COMP_OVERFLOW_BYTES); - if (code) goto _err; - } - - n = tDataTypes[pBlockCol->type].decompFunc(pBuf, pBlockCol->szValue, pSubBlock->nRow, pColData->pData, - pColData->nData, pSubBlock->cmprAlg, *ppBuf, - pColData->nData + COMP_OVERFLOW_BYTES); - if (n < 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _err; - } - - ASSERT(n == pColData->nData); - } - - return code; - -_err: -#endif - return code; -} - -static int32_t tsdbReadSubColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int32_t iSubBlock, - int16_t *aColId, int32_t nCol, SBlockData *pBlockData, uint8_t **ppBuf1, - uint8_t **ppBuf2) { - int32_t code = 0; -#if 0 - TdFilePtr pFD = pReader->pDataFD; - SBlockInfo *pSubBlock = &pBlock->aSubBlock[iSubBlock]; - SArray *aBlockCol = NULL; - int64_t offset; - int64_t size; - int64_t n; - - tBlockDataReset(pBlockData); - pBlockData->nRow = pSubBlock->nRow; - - // TSDBKEY and SBlockCol - if (nCol == 1) { - offset = pSubBlock->offset + pSubBlock->szBlockCol + sizeof(TSCKSUM); - size = pSubBlock->szVersion + pSubBlock->szTSKEY + sizeof(TSCKSUM); - } else { - offset = pSubBlock->offset; - size = pSubBlock->szBlockCol + sizeof(TSCKSUM) + pSubBlock->szVersion + pSubBlock->szTSKEY + sizeof(TSCKSUM); - } - - code = tRealloc(ppBuf1, size); - if (code) goto _err; - - n = taosLSeekFile(pFD, offset, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosReadFile(pFD, *ppBuf1, size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - if (nCol == 1) { - code = tsdbReadBlockDataKey(pBlockData, pSubBlock, *ppBuf1, ppBuf2); - if (code) goto _err; - - goto _exit; - } else { - aBlockCol = taosArrayInit(0, sizeof(SBlockCol)); - if (aBlockCol == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - - code = tsdbReadBlockCol(*ppBuf1, pSubBlock->szBlock, NULL /*todo*/, aBlockCol); - if (code) goto _err; - - code = tsdbReadBlockDataKey(pBlockData, pSubBlock, *ppBuf1 + pSubBlock->szBlockCol + sizeof(TSCKSUM), ppBuf2); - if (code) goto _err; - } - - for (int32_t iCol = 1; iCol < nCol; iCol++) { - void *p = taosArraySearch(aBlockCol, &(SBlockCol){.cid = aColId[iCol]}, tBlockColCmprFn, TD_EQ); - - if (p) { - SBlockCol *pBlockCol = (SBlockCol *)p; - SColData *pColData; - - ASSERT(pBlockCol->flag && pBlockCol->flag != HAS_NONE); - - code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aIdx), &pColData); - if (code) goto _err; - - tColDataInit(pColData, pBlockCol->cid, pBlockCol->type, pBlockCol->smaOn); - if (pBlockCol->flag == HAS_NULL) { - for (int32_t iRow = 0; iRow < pSubBlock->nRow; iRow++) { - code = tColDataAppendValue(pColData, &COL_VAL_NULL(pBlockCol->cid, pBlockCol->type)); - if (code) goto _err; - } - } else { - offset = pSubBlock->offset + pSubBlock->szBlockCol + sizeof(TSCKSUM) + pSubBlock->szVersion + - pSubBlock->szTSKEY + sizeof(TSCKSUM) + pBlockCol->offset; - size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM); - - code = tRealloc(ppBuf1, size); - if (code) goto _err; - - // seek - n = taosLSeekFile(pFD, offset, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // read - n = taosReadFile(pFD, *ppBuf1, size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - code = tsdbReadColDataImpl(pSubBlock, pBlockCol, pColData, *ppBuf1, ppBuf2); - if (code) goto _err; - } - } - } - -_exit: - taosArrayDestroy(aBlockCol); - return code; - -_err: - taosArrayDestroy(aBlockCol); -#endif - return code; -} - -int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int16_t *aColId, int32_t nCol, - SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2) { - int32_t code = 0; - uint8_t *pBuf1 = NULL; - uint8_t *pBuf2 = NULL; - - ASSERT(aColId[0] == PRIMARYKEY_TIMESTAMP_COL_ID); - - if (!ppBuf1) ppBuf1 = &pBuf1; - if (!ppBuf2) ppBuf2 = &pBuf2; - - code = tsdbReadSubColData(pReader, pBlockIdx, pBlock, 0, aColId, nCol, pBlockData, ppBuf1, ppBuf2); - if (code) goto _err; - - if (pBlock->nSubBlock > 1) { - SBlockData *pBlockData1 = &(SBlockData){0}; - SBlockData *pBlockData2 = &(SBlockData){0}; - - tBlockDataCreate(pBlockData1); - tBlockDataCreate(pBlockData2); - for (int32_t iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { - code = tsdbReadSubColData(pReader, pBlockIdx, pBlock, iSubBlock, aColId, nCol, pBlockData1, ppBuf1, ppBuf2); - if (code) goto _err; - - code = tBlockDataCopy(pBlockData, pBlockData2); - if (code) { - tBlockDataDestroy(pBlockData1, 1); - tBlockDataDestroy(pBlockData2, 1); - goto _err; - } - - code = tBlockDataMerge(pBlockData1, pBlockData2, pBlockData); - if (code) { - tBlockDataDestroy(pBlockData1, 1); - tBlockDataDestroy(pBlockData2, 1); - goto _err; - } - } - - tBlockDataDestroy(pBlockData1, 1); - tBlockDataDestroy(pBlockData2, 1); - } - - tFree(pBuf1); - tFree(pBuf2); - return code; - -_err: - tsdbError("vgId:%d, tsdb read col data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf1); - tFree(pBuf2); - return code; -} - -static int32_t tsdbReadSubBlockData(SDataFReader *pReader, SBlock *pBlock, int32_t iSubBlock, SBlockData *pBlockData, - uint8_t **ppBuf1, uint8_t **ppBuf2) { - int32_t code = 0; -#if 0 - uint8_t *p; - int64_t size; - int64_t n; - TdFilePtr pFD = pReader->pDataFD; - SBlockInfo *pSubBlock = &pBlock->aSubBlock[iSubBlock]; - SArray *aBlockCol = NULL; - - tBlockDataReset(pBlockData); - - // realloc - code = tRealloc(ppBuf1, pSubBlock->szBlock); - if (code) goto _err; - - // seek - n = taosLSeekFile(pFD, pSubBlock->offset, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // read - n = taosReadFile(pFD, *ppBuf1, pSubBlock->szBlock); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < pSubBlock->szBlock) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - pBlockData->nRow = pSubBlock->nRow; - - // TSDBKEY - p = *ppBuf1 + pSubBlock->szBlockCol + sizeof(TSCKSUM); - code = tsdbReadBlockDataKey(pBlockData, pSubBlock, p, ppBuf2); - if (code) goto _err; - - // COLUMNS - aBlockCol = taosArrayInit(0, sizeof(SBlockCol)); - if (aBlockCol == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - - code = tsdbReadBlockCol(pSubBlock, *ppBuf1, aBlockCol); - if (code) goto _err; - - for (int32_t iBlockCol = 0; iBlockCol < taosArrayGetSize(aBlockCol); iBlockCol++) { - SColData *pColData; - SBlockCol *pBlockCol = (SBlockCol *)taosArrayGet(aBlockCol, iBlockCol); - - ASSERT(pBlockCol->flag && pBlockCol->flag != HAS_NONE); - - code = tBlockDataAddColData(pBlockData, iBlockCol, &pColData); - if (code) goto _err; - - tColDataInit(pColData, pBlockCol->cid, pBlockCol->type, pBlockCol->smaOn); - if (pBlockCol->flag == HAS_NULL) { - for (int32_t iRow = 0; iRow < pSubBlock->nRow; iRow++) { - code = tColDataAppendValue(pColData, &COL_VAL_NULL(pBlockCol->cid, pBlockCol->type)); - if (code) goto _err; - } - } else { - p = *ppBuf1 + pSubBlock->szBlockCol + sizeof(TSCKSUM) + pSubBlock->szVersion + pSubBlock->szTSKEY + - sizeof(TSCKSUM) + pBlockCol->offset; - code = tsdbReadColDataImpl(pSubBlock, pBlockCol, pColData, p, ppBuf2); - if (code) goto _err; - } - } - - taosArrayDestroy(aBlockCol); - return code; - -_err: - tsdbError("vgId:%d, tsdb read sub block data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); - taosArrayDestroy(aBlockCol); -#endif - return code; -} - -int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, uint8_t **ppBuf1, - uint8_t **ppBuf2) { - int32_t code = 0; - uint8_t *pBuf1 = NULL; - uint8_t *pBuf2 = NULL; - - if (!ppBuf1) ppBuf1 = &pBuf1; - if (!ppBuf2) ppBuf2 = &pBuf2; - - // read the first sub-block - int32_t iSubBlock = 0; - code = tsdbReadSubBlockData(pReader, pBlock, iSubBlock, pBlockData, ppBuf1, ppBuf2); - if (code) goto _err; - - // read remain block data and do merg - if (pBlock->nSubBlock > 1) { - SBlockData *pBlockData1 = &(SBlockData){0}; - SBlockData *pBlockData2 = &(SBlockData){0}; - - tBlockDataCreate(pBlockData1); - tBlockDataCreate(pBlockData2); - for (iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { - code = tsdbReadSubBlockData(pReader, pBlock, iSubBlock, pBlockData1, ppBuf1, ppBuf2); - if (code) { - tBlockDataDestroy(pBlockData1, 1); - tBlockDataDestroy(pBlockData2, 1); - goto _err; - } - - code = tBlockDataCopy(pBlockData, pBlockData2); - if (code) { - tBlockDataDestroy(pBlockData1, 1); - tBlockDataDestroy(pBlockData2, 1); - goto _err; - } - - // merge two block data - code = tBlockDataMerge(pBlockData1, pBlockData2, pBlockData); - if (code) { - tBlockDataDestroy(pBlockData1, 1); - tBlockDataDestroy(pBlockData2, 1); - goto _err; - } - } - - tBlockDataDestroy(pBlockData1, 1); - tBlockDataDestroy(pBlockData2, 1); - } - - ASSERT(pBlock->nRow == pBlockData->nRow); - ASSERT(tsdbKeyCmprFn(&pBlock->minKey, &TSDBROW_KEY(&tBlockDataFirstRow(pBlockData))) == 0); - ASSERT(tsdbKeyCmprFn(&pBlock->maxKey, &TSDBROW_KEY(&tBlockDataLastRow(pBlockData))) == 0); - - tFree(pBuf1); - tFree(pBuf2); - return code; - -_err: - tsdbError("vgId:%d, tsdb read block data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf1); - tFree(pBuf2); - return code; -} - -int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, uint8_t **ppBuf1, - uint8_t **ppBuf2) { - int32_t code = 0; -#if 0 - - tBlockDataReset(pBlockData); - - uint8_t *pBuf1 = NULL; - uint8_t *pBuf2 = NULL; - if (!ppBuf1) ppBuf1 = &pBuf1; - if (!ppBuf2) ppBuf2 = &pBuf2; - - // realloc - code = tRealloc(ppBuf1, pBlockL->szBlock); - if (code) goto _err; - - // seek - int64_t n = taosLSeekFile(pReader->pLastFD, pBlockL->offset, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // read - n = taosReadFile(pReader->pLastFD, *ppBuf1, pBlockL->szBlock); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < pBlockL->szBlock) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // decode block col - SDiskDataHdr hdr; - SArray *aBlockCol = taosArrayInit(0, sizeof(SBlockCol)); - uint8_t *p = *ppBuf1; - if (aBlockCol == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - code = tsdbReadBlockCol(p, pBlockL->szBlockCol, &hdr, aBlockCol); - if (code) goto _err; - p += pBlockL->szBlockCol + sizeof(TSCKSUM); - - // checksum - if (!taosCheckChecksumWhole(p, pBlockL->szUid + pBlockL->szVer + pBlockL->szTSKEY + sizeof(TSCKSUM))) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // UID - if (hdr.uid == 0) { - code = tsdbReadDataArray(p, pBlockL->szUid, pBlockL->nRow, TSDB_DATA_TYPE_BIGINT, pBlockL->cmprAlg, - (uint8_t **)&pBlockData->aUid, ppBuf2); - if (code) goto _err; - } else { - ASSERT(pBlockL->szUid == 0); - } - p += pBlockL->szUid; - - // VERSION - code = tsdbReadDataArray(p, pBlockL->szVer, pBlockL->nRow, TSDB_DATA_TYPE_BIGINT, pBlockL->cmprAlg, - (uint8_t **)&pBlockData->aVersion, ppBuf2); - if (code) goto _err; - p += pBlockL->szVer; - - // TSKEY - code = tsdbReadDataArray(p, pBlockL->szTSKEY, pBlockL->nRow, TSDB_DATA_TYPE_TIMESTAMP, pBlockL->cmprAlg, - (uint8_t **)&pBlockData->aTSKEY, ppBuf2); - if (code) goto _err; - p += pBlockL->szTSKEY; - p += sizeof(TSCKSUM); - - // COLUMN - code = tBlockDataSetSchema(pBlockData, NULL, hdr.suid, hdr.uid); - if (code) goto _err; - - for (int32_t iBlockCol = 0; iBlockCol < taosArrayGetSize(aBlockCol); iBlockCol++) { - SBlockCol *pBlockCol = (SBlockCol *)taosArrayGet(aBlockCol, iBlockCol); - SColData *pColData; - - // checksum - if (!taosCheckChecksumWhole(p, pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM))) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // add SColData - code = tBlockDataAddColData(pBlockData, iBlockCol, &pColData); - if (code) goto _err; - tColDataInit(pColData, pBlockCol->cid, pBlockCol->type, pBlockCol->smaOn); - pColData->nVal = pBlockL->nRow; - pColData->flag = pBlockCol->flag; - - // bitmap - if (pBlockCol->szBitmap) { - code = tsdbReadDataArray(p, pBlockCol->szBitmap, ); - if (code) goto _err; - } - p += pBlockCol->szBitmap; - - // offset - if (pBlockCol->szOffset) { - code = tsdbReadDataArray(p, pBlockCol->szOffset, ); - if (code) goto _err; - } - p += pBlockCol->szOffset; - - // value - pColData->nData = pBlockCol->szOrigin; - if (pColData->nData) { - // TODO - } - } - - taosArrayDestroy(aBlockCol); - - return code; - -_err: - tsdbError("vgId:%d tsdb read last block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); -#endif - return code; -} -#endif - // SDataFWriter ==================================================== int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet) { int32_t code = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index e4ebc3e839..d0d354e467 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -949,13 +949,14 @@ int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) { int32_t size; ASSERT(pColDataSrc->nVal > 0); + ASSERT(pColDataDest->cid = pColDataSrc->cid); + ASSERT(pColDataDest->type = pColDataSrc->type); - pColDataDest->cid = pColDataSrc->cid; - pColDataDest->type = pColDataSrc->type; pColDataDest->smaOn = pColDataSrc->smaOn; pColDataDest->nVal = pColDataSrc->nVal; pColDataDest->flag = pColDataSrc->flag; + // bitmap if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) { size = BIT2_SIZE(pColDataSrc->nVal); code = tRealloc(&pColDataDest->pBitMap, size); @@ -963,6 +964,7 @@ int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) { memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size); } + // offset if (IS_VAR_DATA_TYPE(pColDataDest->type)) { size = sizeof(int32_t) * pColDataSrc->nVal; @@ -972,9 +974,10 @@ int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) { memcpy(pColDataDest->aOffset, pColDataSrc->aOffset, size); } + // value + pColDataDest->nData = pColDataSrc->nData; code = tRealloc(&pColDataDest->pData, pColDataSrc->nData); if (code) goto _exit; - pColDataDest->nData = pColDataSrc->nData; memcpy(pColDataDest->pData, pColDataSrc->pData, pColDataDest->nData); _exit: @@ -1169,32 +1172,25 @@ _exit: return code; } -int32_t tBlockDataInitEx(SBlockData *pBlockData, int64_t *suid, int64_t uid, SArray *aColId) { +int32_t tBlockDataInitEx(SBlockData *pBlockData, SBlockData *pBlockDataFrom) { int32_t code = 0; ASSERT(0); - ASSERT(suid || uid); + ASSERT(pBlockDataFrom->suid || pBlockDataFrom->uid); - pBlockData->suid = suid; - pBlockData->uid = uid; + pBlockData->suid = pBlockDataFrom->suid; + pBlockData->uid = pBlockDataFrom->uid; pBlockData->nRow = 0; taosArrayClear(pBlockData->aIdx); - if (aColId) { - int16_t lcid = -1; - for (int32_t iColId = 0; iColId < taosArrayGetSize(aColId); iColId++) { - int16_t cid = *(int16_t *)taosArrayGet(aColId, iColId); + for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockDataFrom->aIdx); iColData++) { + SColData *pColDataFrom = tBlockDataGetColDataByIdx(pBlockDataFrom, iColData); - ASSERT(cid != PRIMARYKEY_TIMESTAMP_COL_ID); - ASSERT(cid > lcid); - lcid = cid; + SColData *pColData; + code = tBlockDataAddColData(pBlockData, iColData, &pColData); + if (code) goto _exit; - SColData *pColData; - code = tBlockDataAddColData(pBlockData, iColId, &pColData); - if (code) goto _exit; - - tColDataInit(pColData, cid, TSDB_DATA_TYPE_NULL, -1); - } + tColDataInit(pColData, pColDataFrom->cid, pColDataFrom->type, pColDataFrom->smaOn); } _exit: @@ -1339,128 +1335,112 @@ _exit: int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData) { int32_t code = 0; - // set target - int32_t iColData1 = 0; - int32_t nColData1 = taosArrayGetSize(pBlockData1->aIdx); - int32_t iColData2 = 0; - int32_t nColData2 = taosArrayGetSize(pBlockData2->aIdx); - SColData *pColData1; - SColData *pColData2; - SColData *pColData; + ASSERT(pBlockData->suid == pBlockData1->suid); + ASSERT(pBlockData->uid == pBlockData1->uid); + ASSERT(pBlockData1->nRow > 0); + ASSERT(pBlockData2->nRow > 0); - tBlockDataReset(pBlockData); - while (iColData1 < nColData1 && iColData2 < nColData2) { - pColData1 = tBlockDataGetColDataByIdx(pBlockData1, iColData1); - pColData2 = tBlockDataGetColDataByIdx(pBlockData2, iColData2); + tBlockDataClear(pBlockData); - if (pColData1->cid == pColData2->cid) { - code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aIdx), &pColData); - if (code) goto _exit; - tColDataInit(pColData, pColData2->cid, pColData2->type, pColData2->smaOn); + TSDBROW row1 = tsdbRowFromBlockData(pBlockData1, 0); + TSDBROW row2 = tsdbRowFromBlockData(pBlockData2, 0); + TSDBROW *pRow1 = &row1; + TSDBROW *pRow2 = &row2; - iColData1++; - iColData2++; - } else if (pColData1->cid < pColData2->cid) { - code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aIdx), &pColData); - if (code) goto _exit; - tColDataInit(pColData, pColData1->cid, pColData1->type, pColData1->smaOn); + while (pRow1 && pRow2) { + int32_t c = tsdbRowCmprFn(pRow1, pRow2); - iColData1++; - } else { - code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aIdx), &pColData); - if (code) goto _exit; - tColDataInit(pColData, pColData2->cid, pColData2->type, pColData2->smaOn); - - iColData2++; - } - } - - while (iColData1 < nColData1) { - code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aIdx), &pColData); - if (code) goto _exit; - tColDataInit(pColData, pColData1->cid, pColData1->type, pColData1->smaOn); - - iColData1++; - } - - while (iColData2 < nColData2) { - code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aIdx), &pColData); - if (code) goto _exit; - tColDataInit(pColData, pColData2->cid, pColData2->type, pColData2->smaOn); - - iColData2++; - } - - // loop to merge - int32_t iRow1 = 0; - int32_t nRow1 = pBlockData1->nRow; - int32_t iRow2 = 0; - int32_t nRow2 = pBlockData2->nRow; - TSDBROW row1; - TSDBROW row2; - int32_t c; - - while (iRow1 < nRow1 && iRow2 < nRow2) { - row1 = tsdbRowFromBlockData(pBlockData1, iRow1); - row2 = tsdbRowFromBlockData(pBlockData2, iRow2); - - c = tsdbKeyCmprFn(&TSDBROW_KEY(&row1), &TSDBROW_KEY(&row2)); if (c < 0) { - // code = tBlockDataAppendRow(pBlockData, &row1, NULL); + code = tBlockDataAppendRow(pBlockData, pRow1, NULL, + pBlockData1->uid ? pBlockData1->uid : pBlockData1->aUid[pRow1->iRow]); if (code) goto _exit; - iRow1++; + + pRow1->iRow++; + if (pRow1->iRow < pBlockData1->nRow) { + *pRow1 = tsdbRowFromBlockData(pBlockData1, pRow1->iRow); + } else { + pRow1 = NULL; + } } else if (c > 0) { - // code = tBlockDataAppendRow(pBlockData, &row2, NULL); + code = tBlockDataAppendRow(pBlockData, pRow2, NULL, + pBlockData2->uid ? pBlockData2->uid : pBlockData2->aUid[pRow2->iRow]); if (code) goto _exit; - iRow2++; + + pRow2->iRow++; + if (pRow2->iRow < pBlockData2->nRow) { + *pRow2 = tsdbRowFromBlockData(pBlockData2, pRow2->iRow); + } else { + pRow2 = NULL; + } } else { ASSERT(0); } } - while (iRow1 < nRow1) { - row1 = tsdbRowFromBlockData(pBlockData1, iRow1); - // code = tBlockDataAppendRow(pBlockData, &row1, NULL); + while (pRow1) { + code = tBlockDataAppendRow(pBlockData, pRow1, NULL, + pBlockData1->uid ? pBlockData1->uid : pBlockData1->aUid[pRow1->iRow]); if (code) goto _exit; - iRow1++; + + pRow1->iRow++; + if (pRow1->iRow < pBlockData1->nRow) { + *pRow1 = tsdbRowFromBlockData(pBlockData1, pRow1->iRow); + } else { + pRow1 = NULL; + } } - while (iRow2 < nRow2) { - row2 = tsdbRowFromBlockData(pBlockData2, iRow2); - // code = tBlockDataAppendRow(pBlockData, &row2, NULL); + while (pRow2) { + code = tBlockDataAppendRow(pBlockData, pRow2, NULL, + pBlockData2->uid ? pBlockData2->uid : pBlockData2->aUid[pRow2->iRow]); if (code) goto _exit; - iRow2++; + + pRow2->iRow++; + if (pRow2->iRow < pBlockData2->nRow) { + *pRow2 = tsdbRowFromBlockData(pBlockData2, pRow2->iRow); + } else { + pRow2 = NULL; + } } _exit: return code; } -int32_t tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest) { - int32_t code = 0; - SColData *pColDataSrc; - SColData *pColDataDest; +int32_t tBlockDataCopy(SBlockData *pSrc, SBlockData *pDest) { + int32_t code = 0; - ASSERT(pBlockDataSrc->nRow > 0); + tBlockDataClear(pDest); - tBlockDataReset(pBlockDataDest); + ASSERT(pDest->suid == pSrc->suid); + ASSERT(pDest->uid == pSrc->uid); + ASSERT(pSrc->nRow == pDest->nRow); + ASSERT(taosArrayGetSize(pSrc->aIdx) == taosArrayGetSize(pDest->aIdx)); - pBlockDataDest->nRow = pBlockDataSrc->nRow; - // TSDBKEY - code = tRealloc((uint8_t **)&pBlockDataDest->aVersion, sizeof(int64_t) * pBlockDataSrc->nRow); - if (code) goto _exit; - code = tRealloc((uint8_t **)&pBlockDataDest->aTSKEY, sizeof(TSKEY) * pBlockDataSrc->nRow); - if (code) goto _exit; - memcpy(pBlockDataDest->aVersion, pBlockDataSrc->aVersion, sizeof(int64_t) * pBlockDataSrc->nRow); - memcpy(pBlockDataDest->aTSKEY, pBlockDataSrc->aTSKEY, sizeof(TSKEY) * pBlockDataSrc->nRow); + pDest->nRow = pSrc->nRow; - // other - for (size_t iColData = 0; iColData < taosArrayGetSize(pBlockDataSrc->aIdx); iColData++) { - pColDataSrc = tBlockDataGetColDataByIdx(pBlockDataSrc, iColData); - code = tBlockDataAddColData(pBlockDataDest, iColData, &pColDataDest); + if (pSrc->uid == 0) { + code = tRealloc((uint8_t **)&pDest->aUid, sizeof(int64_t) * pDest->nRow); if (code) goto _exit; + memcpy(pDest->aUid, pSrc->aUid, sizeof(int64_t) * pDest->nRow); + } - code = tColDataCopy(pColDataSrc, pColDataDest); + code = tRealloc((uint8_t **)&pDest->aVersion, sizeof(int64_t) * pDest->nRow); + if (code) goto _exit; + memcpy(pDest->aVersion, pSrc->aVersion, sizeof(int64_t) * pDest->nRow); + + code = tRealloc((uint8_t **)&pDest->aTSKEY, sizeof(TSKEY) * pDest->nRow); + if (code) goto _exit; + memcpy(pDest->aTSKEY, pSrc->aTSKEY, sizeof(TSKEY) * pDest->nRow); + + for (int32_t iColData = 0; iColData < taosArrayGetSize(pSrc->aIdx); iColData++) { + SColData *pColSrc = tBlockDataGetColDataByIdx(pSrc, iColData); + SColData *pColDest = tBlockDataGetColDataByIdx(pDest, iColData); + + ASSERT(pColSrc->cid == pColDest->cid); + ASSERT(pColSrc->type == pColDest->type); + + code = tColDataCopy(pColSrc, pColDest); if (code) goto _exit; } From 4f06c4bbddbc64c3d6facc4921aa4576e09d1a3f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 Aug 2022 07:32:02 +0000 Subject: [PATCH 32/92] make it compile --- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 12 ++++++------ source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 2cb3f1f46d..3cff6bef27 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -507,7 +507,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetBlock); /* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); */ - code = tsdbReadDataBlock(state->pDataFReader, &block, state->pBlockData, NULL, 0); + code = tsdbReadDataBlock(state->pDataFReader, &block, state->pBlockData); if (code) goto _err; state->nRow = state->blockData.nRow; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index d48125beb5..108a138d34 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -309,7 +309,7 @@ static int32_t tsdbCommitterNextLastRow(SCommitter *pCommitter) { pCommitter->dReader.iBlockL++; if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { pCommitter->dReader.pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); - code = tsdbReadLastBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockL, pBlockDatal, NULL, 0); + code = tsdbReadLastBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockL, pBlockDatal); if (code) goto _exit; pCommitter->dReader.iRow = 0; @@ -593,7 +593,7 @@ static int32_t tsdbMergeCommitData(SCommitter *pCommitter, STbDataIter *pIter, S SBlockData *pBlockDataR = &pCommitter->dReader.bData; SBlockData *pBlockDataW = &pCommitter->dWriter.bData; - code = tsdbReadDataBlock(pCommitter->dReader.pReader, pBlock, pBlockDataR, NULL, 0); + code = tsdbReadDataBlock(pCommitter->dReader.pReader, pBlock, pBlockDataR); if (code) goto _err; tBlockDataClear(pBlockDataW); @@ -823,8 +823,8 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { // set block data schema if need if (pBlockData->suid == 0 && pBlockData->uid == 0) { - code = tBlockDataSetSchema(pBlockData, pCommitter->skmTable.pTSchema, pTbData->suid, - pTbData->suid ? 0 : pTbData->uid); + code = + tBlockDataInit(pBlockData, pTbData->suid, pTbData->suid ? 0 : pTbData->uid, pCommitter->skmTable.pTSchema); if (code) goto _err; } @@ -966,7 +966,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { if (code) goto _err; tMapDataReset(&pCommitter->dWriter.mBlock); - code = tBlockDataSetSchema(&pCommitter->dWriter.bData, pCommitter->skmTable.pTSchema, pTbData->suid, pTbData->uid); + code = tBlockDataInit(&pCommitter->dWriter.bData, pTbData->suid, pTbData->uid, pCommitter->skmTable.pTSchema); if (code) goto _err; // .data merge @@ -1144,7 +1144,7 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid, 1 /*TOOD*/); if (code) goto _err; - code = tBlockDataSetSchema(pBlockDataW, pCommitter->skmTable.pTSchema, suid, suid ? 0 : uid); + code = tBlockDataInit(pBlockDataW, suid, suid ? 0 : uid, pCommitter->skmTable.pTSchema); if (code) goto _err; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index baafdb488c..73454e2a63 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -785,7 +785,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBlockIdx blockIdx = {.suid = pReader->suid, .uid = pBlockScanInfo->uid}; - int32_t code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData, pSupInfo->colIds, numOfCols); + int32_t code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData /*, pSupInfo->colIds, numOfCols*/); if (code != TSDB_CODE_SUCCESS) { goto _error; } From f37a072e77e44ab0bba19713c4dad493b4350860 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 Aug 2022 10:06:07 +0000 Subject: [PATCH 33/92] more work --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 147 +++++++++++++---------- 1 file changed, 81 insertions(+), 66 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 108a138d34..2395a4cd8b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -53,7 +53,6 @@ typedef struct { // last SArray *aBlockL; // SArray int32_t iBlockL; - SBlockL *pBlockL; SBlockData bDatal; int32_t iRow; SRowInfo *pRowInfo; @@ -290,6 +289,54 @@ _err: return code; } +static int32_t tsdbCommitterUpdateTableSchema(SCommitter *pCommitter, int64_t suid, int64_t uid, int32_t sver) { + int32_t code = 0; + + if (pCommitter->skmTable.pTSchema) { + if (pCommitter->skmTable.suid == suid) { + if (suid == 0) { + if (pCommitter->skmTable.uid == uid && sver == pCommitter->skmTable.pTSchema->version) goto _exit; + } else { + if (sver == pCommitter->skmTable.pTSchema->version) goto _exit; + } + } + } + + pCommitter->skmTable.suid = suid; + pCommitter->skmTable.uid = uid; + tTSchemaDestroy(pCommitter->skmTable.pTSchema); + code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmTable.pTSchema); + if (code) goto _exit; + +_exit: + return code; +} + +static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid, int64_t uid, int32_t sver) { + int32_t code = 0; + + if (pCommitter->skmRow.pTSchema) { + if (pCommitter->skmRow.suid == suid) { + if (suid == 0) { + if (pCommitter->skmRow.uid == uid && sver == pCommitter->skmRow.pTSchema->version) goto _exit; + } else { + if (sver == pCommitter->skmRow.pTSchema->version) goto _exit; + } + } + } + + pCommitter->skmRow.suid = suid; + pCommitter->skmRow.uid = uid; + tTSchemaDestroy(pCommitter->skmRow.pTSchema); + code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmRow.pTSchema); + if (code) { + goto _exit; + } + +_exit: + return code; +} + static int32_t tsdbCommitterNextLastRow(SCommitter *pCommitter) { int32_t code = 0; @@ -308,8 +355,17 @@ static int32_t tsdbCommitterNextLastRow(SCommitter *pCommitter) { } else { pCommitter->dReader.iBlockL++; if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { - pCommitter->dReader.pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); - code = tsdbReadLastBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockL, pBlockDatal); + SBlockL *pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); + int64_t suid = pBlockL->suid; + int64_t uid = pBlockL->maxUid; + + code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid, 1 /*TODO*/); + if (code) goto _exit; + + code = tBlockDataInit(pBlockDatal, suid, suid ? 0 : uid, pCommitter->skmTable.pTSchema); + if (code) goto _exit; + + code = tsdbReadLastBlock(pCommitter->dReader.pReader, pBlockL, pBlockDatal); if (code) goto _exit; pCommitter->dReader.iRow = 0; @@ -444,54 +500,6 @@ _err: return code; } -static int32_t tsdbCommitterUpdateTableSchema(SCommitter *pCommitter, int64_t suid, int64_t uid, int32_t sver) { - int32_t code = 0; - - if (pCommitter->skmTable.pTSchema) { - if (pCommitter->skmTable.suid == suid) { - if (suid == 0) { - if (pCommitter->skmTable.uid == uid && sver == pCommitter->skmTable.pTSchema->version) goto _exit; - } else { - if (sver == pCommitter->skmTable.pTSchema->version) goto _exit; - } - } - } - - pCommitter->skmTable.suid = suid; - pCommitter->skmTable.uid = uid; - tTSchemaDestroy(pCommitter->skmTable.pTSchema); - code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmTable.pTSchema); - if (code) goto _exit; - -_exit: - return code; -} - -static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid, int64_t uid, int32_t sver) { - int32_t code = 0; - - if (pCommitter->skmRow.pTSchema) { - if (pCommitter->skmRow.suid == suid) { - if (suid == 0) { - if (pCommitter->skmRow.uid == uid && sver == pCommitter->skmRow.pTSchema->version) goto _exit; - } else { - if (sver == pCommitter->skmRow.pTSchema->version) goto _exit; - } - } - } - - pCommitter->skmRow.suid = suid; - pCommitter->skmRow.uid = uid; - tTSchemaDestroy(pCommitter->skmRow.pTSchema); - code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmRow.pTSchema); - if (code) { - goto _exit; - } - -_exit: - return code; -} - static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) { int32_t code = 0; SBlockData *pBlockData = &pCommitter->dWriter.bData; @@ -779,13 +787,13 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { STbData *pTbData = pIter->pTbData; int32_t nRow = tsdbGetNumOfRowsLessThan(pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = VERSION_MIN}); - if (pCommitter->dReader.pRowInfo) { - if (pCommitter->dReader.pRowInfo->suid) { + if (pCommitter->dReader.pRowInfo && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pRowInfo) == 0) { + if (pCommitter->dReader.pRowInfo->suid) { // super table for (int32_t iRow = pCommitter->dReader.iRow; iRow < pCommitter->dReader.bDatal.nRow; iRow++) { if (pTbData->uid != pCommitter->dReader.bDatal.aUid[iRow]) break; nRow++; } - } else { + } else { // normal table ASSERT(pCommitter->dReader.iRow == 0); nRow += pCommitter->dReader.bDatal.nRow; } @@ -812,10 +820,13 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { pBlockData = &pCommitter->dWriter.bDatal; // commit and reset block data schema if need - if (pBlockData->nRow > 0) { - if (pBlockData->suid != pTbData->suid || pBlockData->suid == 0) { - code = tsdbCommitLastBlock(pCommitter); - if (code) goto _err; + // QUESTION: Is there a case that pBlockData->nRow == 0 but need to change schema ? + if (pBlockData->suid || pBlockData->uid) { + if (pBlockData->suid != pTbData->uid || pBlockData->suid == 0) { + if (pBlockData->nRow > 0) { + code = tsdbCommitLastBlock(pCommitter); + if (code) goto _err; + } tBlockDataReset(pBlockData); } @@ -962,10 +973,12 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { pBlock = NULL; } - code = tsdbCommitterUpdateTableSchema(pCommitter, pTbData->suid, pTbData->uid, pTbData->maxSkmVer); + code = tsdbCommitterUpdateTableSchema(pCommitter, pTbData->suid, pTbData->uid, pTbData->maxSkmVer /*TODO*/); if (code) goto _err; tMapDataReset(&pCommitter->dWriter.mBlock); + code = tBlockDataInit(&pCommitter->dReader.bData, pTbData->suid, pTbData->uid, pCommitter->skmTable.pTSchema); + if (code) goto _err; code = tBlockDataInit(&pCommitter->dWriter.bData, pTbData->suid, pTbData->uid, pCommitter->skmTable.pTSchema); if (code) goto _err; @@ -1130,11 +1143,12 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { ASSERT(pBlockDataR->nRow > 0); // commit and reset block data schema if need - if (pBlockDataW->nRow > 0) { - if (pBlockDataW->suid != pCommitter->dReader.pRowInfo->suid || pBlockDataW->suid == 0) { - code = tsdbCommitLastBlock(pCommitter); - if (code) goto _err; - + if (pBlockDataW->suid || pBlockDataW->uid) { + if (pBlockDataW->suid != suid || pBlockDataW->suid == 0) { + if (pBlockDataW->nRow > 0) { + code = tsdbCommitLastBlock(pCommitter); + if (code) goto _err; + } tBlockDataReset(pBlockDataW); } } @@ -1151,9 +1165,10 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { // check if it can make sure that one table data in one block int32_t nRow = 0; if (pBlockDataR->suid) { - for (int32_t iRow = pCommitter->dReader.iRow; (iRow < pBlockDataR->nRow) && (pBlockDataR->aUid[iRow] == uid); - iRow++) { + int32_t iRow = pCommitter->dReader.iRow; + while ((iRow < pBlockDataR->nRow) && (pBlockDataR->aUid[iRow] == uid)) { nRow++; + iRow++; } } else { ASSERT(pCommitter->dReader.iRow == 0); From 0e06d86f7bac208b03d456653eae05e513a9a1dc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 Aug 2022 11:40:49 +0000 Subject: [PATCH 34/92] fix some bug --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 5 ++--- source/dnode/vnode/src/tsdb/tsdbUtil.c | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 0e10d1b000..f46726e647 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -187,7 +187,7 @@ int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, ui int32_t *szOut, uint8_t **ppBuf); int32_t tsdbDecmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t szOut, uint8_t **ppBuf); -int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppOut, int8_t nOut, +int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppOut, int32_t nOut, uint8_t **ppBuf); int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, int32_t nVal, SColData *pColData, uint8_t **ppBuf); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index f73883d050..af1342daf3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1050,11 +1050,10 @@ _err: int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { int32_t code = 0; - STsdb *pTsdb = (*ppWriter)->pTsdb; + STsdb *pTsdb = NULL; if (*ppWriter == NULL) goto _exit; - - if (sync) { + pTsdb = (*ppWriter)->pTsdb if (sync) { if (taosFsyncFile((*ppWriter)->pHeadFD) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index d0d354e467..d5bd1f8c0c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1794,7 +1794,7 @@ _exit: return code; } -int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppOut, int8_t nOut, +int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppOut, int32_t nOut, uint8_t **ppBuf) { int32_t code = 0; From 50d03237ed07110b841b46fb8f1bd2f96de87163 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 Aug 2022 12:14:16 +0000 Subject: [PATCH 35/92] fix more bug --- source/dnode/vnode/src/tsdb/tsdbFS.c | 17 ++++------------- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 7 +++++-- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 74f1aef1fc..247de99338 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -576,10 +576,7 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - fSet.pHeadF->nRef = 0; - fSet.pHeadF->commitID = pSet->pHeadF->commitID; - fSet.pHeadF->size = pSet->pHeadF->size; - fSet.pHeadF->offset = pSet->pHeadF->offset; + *fSet.pHeadF = *pSet->pHeadF; // data fSet.pDataF = (SDataFile *)taosMemoryMalloc(sizeof(SDataFile)); @@ -587,9 +584,7 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - fSet.pDataF->nRef = 0; - fSet.pDataF->commitID = pSet->pDataF->commitID; - fSet.pDataF->size = pSet->pDataF->size; + *fSet.pDataF = *pSet->pDataF; // data fSet.pLastF = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); @@ -597,9 +592,7 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - fSet.pLastF->nRef = 0; - fSet.pLastF->commitID = pSet->pLastF->commitID; - fSet.pLastF->size = pSet->pLastF->size; + *fSet.pLastF = *pSet->pLastF; // last fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); @@ -607,9 +600,7 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - fSet.pSmaF->nRef = 0; - fSet.pSmaF->commitID = pSet->pSmaF->commitID; - fSet.pSmaF->size = pSet->pSmaF->size; + *fSet.pSmaF = *pSet->pSmaF; if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index af1342daf3..06aa5ef3cb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -63,6 +63,7 @@ _err: int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync) { int32_t code = 0; SDelFWriter *pWriter = *ppWriter; + STsdb *pTsdb = pWriter->pTsdb; // sync if (sync && taosFsyncFile(pWriter->pWriteH) < 0) { @@ -83,7 +84,7 @@ int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync) { return code; _err: - tsdbError("vgId:%d, failed to close del file writer since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, failed to close del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } @@ -1053,7 +1054,9 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { STsdb *pTsdb = NULL; if (*ppWriter == NULL) goto _exit; - pTsdb = (*ppWriter)->pTsdb if (sync) { + + pTsdb = (*ppWriter)->pTsdb; + if (sync) { if (taosFsyncFile((*ppWriter)->pHeadFD) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; From 78985ce4f1cca997c19d932ffce04d2ac7d99d75 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 Aug 2022 13:05:21 +0000 Subject: [PATCH 36/92] fix more bug --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 6 ++++++ source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 13 +++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 2395a4cd8b..5d30599881 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1222,6 +1222,12 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) { // commit current table data code = tsdbCommitTableData(pCommitter, pTbData); if (code) goto _err; + + // move next reader table data if need + if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pBlockIdx) == 0) { + code = tsdbCommitterNextTableData(pCommitter); + if (code) goto _err; + } } code = tsdbMoveCommitData(pCommitter, (TABLEID){.suid = INT64_MAX, .uid = INT64_MAX}); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 06aa5ef3cb..fbdcd0f053 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1237,6 +1237,8 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) { pHeadFile->size += size; _exit: + tsdbTrace("vgId:%d write block idx, offset:%" PRId64 " size:%" PRId64 " nBlockIdx:%d", TD_VID(pWriter->pTsdb->pVnode), + pHeadFile->offset, size, taosArrayGetSize(aBlockIdx)); return code; _err: @@ -1277,8 +1279,10 @@ int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, SBlockIdx *pBloc pBlockIdx->size = size; pHeadFile->size += size; - tsdbTrace("vgId:%d, write block, offset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), pBlockIdx->offset, - pBlockIdx->size); + tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%d suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64 + " size:%" PRId64 " nItem:%d", + TD_VID(pWriter->pTsdb->pVnode), pWriter->wSet.fid, pHeadFile->commitID, pBlockIdx->suid, pBlockIdx->uid, + pBlockIdx->offset, pBlockIdx->size, mBlock->nItem); return code; _err: @@ -1331,6 +1335,8 @@ int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) { pHeadFile->size += size; _exit: + tsdbTrace("vgId:%d tsdb write blockl, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), + pHeadFile->loffset, size); return code; _err: @@ -1542,6 +1548,9 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock } _exit: + tsdbTrace("vgId:%d tsdb write block data, suid:%" PRId64 " uid:%" PRId64 " nRow:%d, offset:%" PRId64 " size:%d", + TD_VID(pWriter->pTsdb->pVnode), pBlockData->suid, pBlockData->uid, pBlockData->nRow, pBlkInfo->offset, + pBlkInfo->szBlock); return code; _err: From 4ac14697b781a83292bb1fdd428306bb1a48ee97 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 Aug 2022 13:23:50 +0000 Subject: [PATCH 37/92] fix more bug --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 5d30599881..ae7c5f1a90 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -822,7 +822,7 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { // commit and reset block data schema if need // QUESTION: Is there a case that pBlockData->nRow == 0 but need to change schema ? if (pBlockData->suid || pBlockData->uid) { - if (pBlockData->suid != pTbData->uid || pBlockData->suid == 0) { + if (pBlockData->suid != pTbData->suid || pBlockData->suid == 0) { if (pBlockData->nRow > 0) { code = tsdbCommitLastBlock(pCommitter); if (code) goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index fbdcd0f053..69a4227486 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -771,7 +771,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo if (code) goto _err; p += hdr.szKey; - ASSERT(p - pReader->pBuf1 == sizeof(TSCKSUM)); + ASSERT(p - pReader->pBuf1 == pBlkInfo->szKey - sizeof(TSCKSUM)); // read and decode columns if (taosArrayGetSize(pBlockData->aIdx) == 0) goto _exit; @@ -1195,7 +1195,7 @@ _err: int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) { int32_t code = 0; SHeadFile *pHeadFile = &pWriter->fHead; - int64_t size; + int64_t size = 0; int64_t n; // check From 5938ec6a7700bde07799497f4d18fa588c6bfaa5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 15 Aug 2022 08:44:41 +0000 Subject: [PATCH 38/92] more work --- source/util/src/trbtree.c | 210 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 source/util/src/trbtree.c diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c new file mode 100644 index 0000000000..0970485dad --- /dev/null +++ b/source/util/src/trbtree.c @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" + +typedef int32_t (*tRBTreeCmprFn)(void *, void *); + +typedef struct SRBTree SRBTree; +typedef struct SRBTreeNode SRBTreeNode; +typedef struct SRBTreeIter SRBTreeIter; + +struct SRBTreeNode { + enum { RED, BLACK } color; + SRBTreeNode *parent; + SRBTreeNode *left; + SRBTreeNode *right; + uint8_t payload[]; +}; + +struct SRBTree { + tRBTreeCmprFn cmprFn; + SRBTreeNode *root; +}; + +struct SRBTreeIter { + SRBTree *pTree; +}; + +#define RBTREE_NODE_COLOR(N) ((N) ? (N)->color : BLACK) + +// APIs ================================================ +static void tRBTreeRotateLeft(SRBTree *pTree, SRBTreeNode *pNode) { + SRBTreeNode *right = pNode->right; + + pNode->right = right->left; + if (pNode->right) { + pNode->right->parent = pNode; + } + + right->parent = pNode->parent; + if (pNode->parent == NULL) { + pTree->root = right; + } else if (pNode == pNode->parent->left) { + pNode->parent->left = right; + } else { + pNode->parent->right = right; + } + + right->left = pNode; + pNode->parent = right; +} + +static void tRBTreeRotateRight(SRBTree *pTree, SRBTreeNode *pNode) { + SRBTreeNode *left = pNode->left; + + pNode->left = left->right; + if (pNode->left) { + pNode->left->parent = pNode; + } + + left->parent = pNode->parent; + if (pNode->parent == NULL) { + pTree->root = left; + } else if (pNode == pNode->parent->left) { + pNode->parent->left = left; + } else { + pNode->parent->right = left; + } + + left->right = pNode; + pNode->parent = left; +} + +#define tRBTreeCreate(compare) \ + (SRBTree) { .cmprFn = (compare), .root = NULL } + +SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *pNew) { + pNew->left = NULL; + pNew->right = NULL; + pNew->color = RED; + + // insert + if (pTree->root == NULL) { + pNew->parent = NULL; + pTree->root = pNew; + } else { + SRBTreeNode *pNode = pTree->root; + while (true) { + ASSERT(pNode); + + int32_t c = pTree->cmprFn(pNew->payload, pNode->payload); + if (c < 0) { + if (pNode->left) { + pNode = pNode->left; + } else { + pNew->parent = pNode; + pNode->left = pNew; + break; + } + } else if (c > 0) { + if (pNode->right) { + pNode = pNode->right; + } else { + pNew->parent = pNode; + pNode->right = pNew; + break; + } + } else { + return NULL; + } + } + } + + // fix + SRBTreeNode *pNode = pNew; + while (pNode->parent && pNode->parent->color == RED) { + SRBTreeNode *p = pNode->parent; + SRBTreeNode *g = p->parent; + + if (p == g->left) { + SRBTreeNode *u = g->right; + + if (RBTREE_NODE_COLOR(u) == RED) { + p->color = BLACK; + u->color = BLACK; + g->color = RED; + pNode = g; + } else { + if (pNode == p->right) { + pNode = p; + tRBTreeRotateLeft(pTree, pNode); + } + pNode->parent->color = BLACK; + pNode->parent->parent->color = RED; + tRBTreeRotateRight(pTree, pNode->parent->parent); + } + } else { + SRBTreeNode *u = g->left; + + if (RBTREE_NODE_COLOR(u) == RED) { + p->color = BLACK; + u->color = BLACK; + g->color = RED; + } else { + if (pNode == p->left) { + pNode = p; + tRBTreeRotateRight(pTree, pNode); + } + pNode->parent->color = BLACK; + pNode->parent->parent->color = RED; + tRBTreeRotateLeft(pTree, pNode->parent->parent); + } + } + } + + pTree->root->color = BLACK; + return pNew; +} + +SRBTreeNode *tRBTreeDrop(SRBTree *pTree, void *pKey) { + SRBTreeNode *pNode = pTree->root; + + while (pNode) { + int32_t c = pTree->cmprFn(pKey, pNode->payload); + + if (c < 0) { + pNode = pNode->left; + } else if (c > 0) { + pNode = pNode->right; + } else { + break; + } + } + + if (pNode) { + // TODO + } + + return pNode; +} + +SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey) { + SRBTreeNode *pNode = pTree->root; + + while (pNode) { + int32_t c = pTree->cmprFn(pKey, pNode->payload); + + if (c < 0) { + pNode = pNode->left; + } else if (c > 0) { + pNode = pNode->right; + } else { + break; + } + } + + return pNode; +} From 3359b8e6204a02b5d3365c7a24ff44767f28df08 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 15 Aug 2022 10:16:07 +0000 Subject: [PATCH 39/92] more optimize --- source/dnode/vnode/src/inc/tsdb.h | 18 ++- source/dnode/vnode/src/tsdb/tsdbCache.c | 4 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 19 ++- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 144 +++++++++++++++------ source/dnode/vnode/src/tsdb/tsdbRead.c | 8 +- 5 files changed, 134 insertions(+), 59 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index f46726e647..0a03334cfd 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -194,11 +194,12 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in int32_t tsdbReadAndCheck(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck); // tsdbMemTable ============================================================================================== // SMemTable -int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); -void tsdbMemTableDestroy(SMemTable *pMemTable); -void tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData); -void tsdbRefMemTable(SMemTable *pMemTable); -void tsdbUnrefMemTable(SMemTable *pMemTable); +int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); +void tsdbMemTableDestroy(SMemTable *pMemTable); +STbData *tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid); +void tsdbRefMemTable(SMemTable *pMemTable); +void tsdbUnrefMemTable(SMemTable *pMemTable); +SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable); // STbDataIter int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter); void *tsdbTbDataIterDestroy(STbDataIter *pIter); @@ -359,6 +360,7 @@ struct STbData { SDelData *pHead; SDelData *pTail; SMemSkipList sl; + STbData *next; }; struct SMemTable { @@ -372,7 +374,11 @@ struct SMemTable { int64_t maxVersion; int64_t nRow; int64_t nDel; - SArray *aTbData; // SArray + struct { + int32_t nTbData; + int32_t nBucket; + STbData **aBucket; + }; }; struct TSDBROW { diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 3cff6bef27..24d6b2f385 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -742,12 +742,12 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs STbData *pMem = NULL; if (pIter->pReadSnap->pMem) { - tsdbGetTbDataFromMemTable(pIter->pReadSnap->pMem, suid, uid, &pMem); + pMem = tsdbGetTbDataFromMemTable(pIter->pReadSnap->pMem, suid, uid); } STbData *pIMem = NULL; if (pIter->pReadSnap->pIMem) { - tsdbGetTbDataFromMemTable(pIter->pReadSnap->pIMem, suid, uid, &pIMem); + pIMem = tsdbGetTbDataFromMemTable(pIter->pReadSnap->pIMem, suid, uid); } pIter->pTsdb = pTsdb; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index ae7c5f1a90..f59a8e53f1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -35,6 +35,7 @@ typedef struct { int32_t minRow; int32_t maxRow; int8_t cmprAlg; + SArray *aTbDataP; STsdbFS fs; // -------------- TSKEY nextKey; // reset by each table commit @@ -1212,8 +1213,8 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) { if (code) goto _err; // commit file data impl - for (int32_t iTbData = 0; iTbData < taosArrayGetSize(pMemTable->aTbData); iTbData++) { - STbData *pTbData = (STbData *)taosArrayGetP(pMemTable->aTbData, iTbData); + for (int32_t iTbData = 0; iTbData < taosArrayGetSize(pCommitter->aTbDataP); iTbData++) { + STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData); // move commit until current (suid, uid) code = tsdbMoveCommitData(pCommitter, *(TABLEID *)pTbData); @@ -1270,6 +1271,11 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { pCommitter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows; pCommitter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows; pCommitter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; + pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem); + if (pCommitter->aTbDataP == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } code = tsdbFSCopy(pTsdb, &pCommitter->fs); if (code) goto _err; @@ -1395,13 +1401,13 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { int32_t iDelIdx = 0; int32_t nDelIdx = taosArrayGetSize(pCommitter->aDelIdx); int32_t iTbData = 0; - int32_t nTbData = taosArrayGetSize(pMemTable->aTbData); + int32_t nTbData = taosArrayGetSize(pCommitter->aTbDataP); STbData *pTbData; SDelIdx *pDelIdx; ASSERT(nTbData > 0); - pTbData = (STbData *)taosArrayGetP(pMemTable->aTbData, iTbData); + pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData); pDelIdx = (iDelIdx < nDelIdx) ? (SDelIdx *)taosArrayGet(pCommitter->aDelIdx, iDelIdx) : NULL; while (true) { if (pTbData == NULL && pDelIdx == NULL) break; @@ -1427,7 +1433,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { if (code) goto _err; iTbData++; - pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pMemTable->aTbData, iTbData) : NULL; + pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL; continue; _commit_disk_del: @@ -1443,7 +1449,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { if (code) goto _err; iTbData++; - pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pMemTable->aTbData, iTbData) : NULL; + pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL; iDelIdx++; pDelIdx = (iDelIdx < nDelIdx) ? (SDelIdx *)taosArrayGet(pCommitter->aDelIdx, iDelIdx) : NULL; continue; @@ -1491,6 +1497,7 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { tsdbUnrefMemTable(pMemTable); tsdbFSDestroy(&pCommitter->fs); + taosArrayDestroy(pCommitter->aTbDataP); tsdbInfo("vgId:%d, tsdb end commit", TD_VID(pTsdb->pVnode)); return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 8ae0e824cf..278b657b9e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -15,6 +15,7 @@ #include "tsdb.h" +#define MEM_MIN_HASH 1024 #define SL_MAX_LEVEL 5 #define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2) @@ -49,8 +50,10 @@ int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { pMemTable->maxVersion = VERSION_MIN; pMemTable->nRow = 0; pMemTable->nDel = 0; - pMemTable->aTbData = taosArrayInit(128, sizeof(STbData *)); - if (pMemTable->aTbData == NULL) { + pMemTable->nTbData = 0; + pMemTable->nBucket = MEM_MIN_HASH; + pMemTable->aBucket = (STbData **)taosMemoryCalloc(pMemTable->nBucket, sizeof(STbData *)); + if (pMemTable->aBucket == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pMemTable); goto _err; @@ -68,37 +71,30 @@ _err: void tsdbMemTableDestroy(SMemTable *pMemTable) { if (pMemTable) { vnodeBufPoolUnRef(pMemTable->pPool); - taosArrayDestroy(pMemTable->aTbData); + taosMemoryFree(pMemTable->aBucket); taosMemoryFree(pMemTable); } } -static int32_t tbDataPCmprFn(const void *p1, const void *p2) { - STbData *pTbData1 = *(STbData **)p1; - STbData *pTbData2 = *(STbData **)p2; +static FORCE_INLINE STbData *tsdbGetTbDataFromMemTableImpl(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid) { + STbData *pTbData = pMemTable->aBucket[TABS(uid) % pMemTable->nBucket]; - if (pTbData1->suid < pTbData2->suid) { - return -1; - } else if (pTbData1->suid > pTbData2->suid) { - return 1; + while (pTbData) { + if (pTbData->uid == uid) break; + pTbData = pTbData->next; } - if (pTbData1->uid < pTbData2->uid) { - return -1; - } else if (pTbData1->uid > pTbData2->uid) { - return 1; - } - - return 0; + return pTbData; } -void tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData) { - STbData *pTbData = &(STbData){.suid = suid, .uid = uid}; + +STbData *tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid) { + STbData *pTbData; taosRLockLatch(&pMemTable->latch); - void *p = taosArraySearch(pMemTable->aTbData, &pTbData, tbDataPCmprFn, TD_EQ); + pTbData = tsdbGetTbDataFromMemTableImpl(pMemTable, suid, uid); taosRUnLockLatch(&pMemTable->latch); - *ppTbData = p ? *(STbData **)p : NULL; + return pTbData; } int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, @@ -318,18 +314,44 @@ _exit: return pIter->pRow; } +static int32_t tsdbMemTableRehash(SMemTable *pMemTable) { + int32_t code = 0; + + int32_t nBucket = pMemTable->nBucket * 2; + STbData **aBucket = (STbData **)taosMemoryCalloc(nBucket, sizeof(STbData *)); + if (aBucket == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t iBucket = 0; iBucket < pMemTable->nBucket; iBucket++) { + STbData *pTbData = pMemTable->aBucket[iBucket]; + + while (pTbData) { + STbData *pNext = pTbData->next; + + int32_t idx = TABS(pTbData->uid) % nBucket; + pTbData->next = aBucket[idx]; + aBucket[idx] = pTbData; + + pTbData = pNext; + } + } + + taosMemoryFree(pMemTable->aBucket); + pMemTable->nBucket = nBucket; + pMemTable->aBucket = aBucket; + +_exit: + return code; +} + static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData) { - int32_t code = 0; - int32_t idx = 0; - STbData *pTbData = NULL; - STbData *pTbDataT = &(STbData){.suid = suid, .uid = uid}; + int32_t code = 0; // get - idx = taosArraySearchIdx(pMemTable->aTbData, &pTbDataT, tbDataPCmprFn, TD_GE); - if (idx >= 0) { - pTbData = (STbData *)taosArrayGetP(pMemTable->aTbData, idx); - if (tbDataPCmprFn(&pTbDataT, &pTbData) == 0) goto _exit; - } + STbData *pTbData = tsdbGetTbDataFromMemTableImpl(pMemTable, suid, uid); + if (pTbData) goto _exit; // create SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse; @@ -365,21 +387,23 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid SL_NODE_FORWARD(pTbData->sl.pTail, iLevel) = NULL; } - void *p; - if (idx < 0) { - idx = taosArrayGetSize(pMemTable->aTbData); + taosWLockLatch(&pMemTable->latch); + + if (pMemTable->nTbData >= pMemTable->nBucket) { + code = tsdbMemTableRehash(pMemTable); + if (code) { + taosWUnLockLatch(&pMemTable->latch); + goto _err; + } } - taosWLockLatch(&pMemTable->latch); - p = taosArrayInsert(pMemTable->aTbData, idx, &pTbData); + int32_t idx = TABS(uid) % pMemTable->nBucket; + pTbData->next = pMemTable->aBucket[idx]; + pMemTable->aBucket[idx] = pTbData; + pMemTable->nTbData++; + taosWUnLockLatch(&pMemTable->latch); - tsdbDebug("vgId:%d, add table data %p at idx:%d", TD_VID(pMemTable->pTsdb->pVnode), pTbData, idx); - - if (p == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } _exit: *ppTbData = pTbData; return code; @@ -622,3 +646,41 @@ void tsdbUnrefMemTable(SMemTable *pMemTable) { tsdbMemTableDestroy(pMemTable); } } + +static FORCE_INLINE int32_t tbDataPCmprFn(const void *p1, const void *p2) { + STbData *pTbData1 = *(STbData **)p1; + STbData *pTbData2 = *(STbData **)p2; + + if (pTbData1->suid < pTbData2->suid) { + return -1; + } else if (pTbData1->suid > pTbData2->suid) { + return 1; + } + + if (pTbData1->uid < pTbData2->uid) { + return -1; + } else if (pTbData1->uid > pTbData2->uid) { + return 1; + } + + return 0; +} + +SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable) { + SArray *aTbDataP = taosArrayInit(pMemTable->nTbData, sizeof(STbData *)); + if (aTbDataP == NULL) goto _exit; + + for (int32_t iBucket = 0; iBucket < pMemTable->nBucket; iBucket++) { + STbData *pTbData = pMemTable->aBucket[iBucket]; + + while (pTbData) { + taosArrayPush(aTbDataP, &pTbData); + pTbData = pTbData->next; + } + } + + taosArraySort(aTbDataP, tbDataPCmprFn); + +_exit: + return aTbDataP; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c6ae1c529c..3e2da64701 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1616,7 +1616,7 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea STbData* d = NULL; if (pReader->pReadSnap->pMem != NULL) { - tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->suid, pBlockScanInfo->uid, &d); + d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->suid, pBlockScanInfo->uid); if (d != NULL) { code = tsdbTbDataIterCreate(d, &startKey, backward, &pBlockScanInfo->iter.iter); if (code == TSDB_CODE_SUCCESS) { @@ -1637,7 +1637,7 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea STbData* di = NULL; if (pReader->pReadSnap->pIMem != NULL) { - tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->suid, pBlockScanInfo->uid, &di); + di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->suid, pBlockScanInfo->uid); if (di != NULL) { code = tsdbTbDataIterCreate(di, &startKey, backward, &pBlockScanInfo->iiter.iter); if (code == TSDB_CODE_SUCCESS) { @@ -3103,7 +3103,7 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { STbData* d = NULL; if (pReader->pTsdb->mem != NULL) { - tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->suid, pBlockScanInfo->uid, &d); + d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->suid, pBlockScanInfo->uid); if (d != NULL) { rows += tsdbGetNRowsInTbData(d); } @@ -3111,7 +3111,7 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { STbData* di = NULL; if (pReader->pTsdb->imem != NULL) { - tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->suid, pBlockScanInfo->uid, &di); + di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->suid, pBlockScanInfo->uid); if (di != NULL) { rows += tsdbGetNRowsInTbData(di); } From 43c84c977aa8b9897c56ea30df4baef3a92bbf08 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 16 Aug 2022 01:45:37 +0000 Subject: [PATCH 40/92] adjust more --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 8 +++---- source/dnode/vnode/src/tsdb/tsdbFile.c | 4 ++-- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 22 +++++++++---------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 0a03334cfd..7e88a884c5 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -540,7 +540,6 @@ struct SHeadFile { int64_t commitID; int64_t size; int64_t offset; - int64_t loffset; }; struct SDataFile { @@ -555,6 +554,7 @@ struct SLastFile { int64_t commitID; int64_t size; + int64_t offset; }; struct SSmaFile { diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index f59a8e53f1..589bce9aa5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -466,9 +466,9 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { if (pRSet) { wSet.diskId = pRSet->diskId; wSet.fid = pCommitter->commitFid; - fHead = (SHeadFile){.commitID = pCommitter->commitID, .offset = 0, .size = 0, .loffset = 0}; + fHead = (SHeadFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0}; fData = *pRSet->pDataF; - fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0}; + fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0}; fSma = *pRSet->pSmaF; } else { SDiskID did = {0}; @@ -479,9 +479,9 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { wSet.diskId = did; wSet.fid = pCommitter->commitFid; - fHead = (SHeadFile){.commitID = pCommitter->commitID, .offset = 0, .size = 0, .loffset = 0}; + fHead = (SHeadFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0}; fData = (SDataFile){.commitID = pCommitter->commitID, .size = 0}; - fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0}; + fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0}; fSma = (SSmaFile){.commitID = pCommitter->commitID, .size = 0}; } code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet); diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 4043cb7470..5db02fa285 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -21,7 +21,6 @@ int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile) { n += tPutI64v(p ? p + n : p, pHeadFile->commitID); n += tPutI64v(p ? p + n : p, pHeadFile->size); n += tPutI64v(p ? p + n : p, pHeadFile->offset); - n += tPutI64v(p ? p + n : p, pHeadFile->loffset); return n; } @@ -32,7 +31,6 @@ static int32_t tGetHeadFile(uint8_t *p, SHeadFile *pHeadFile) { n += tGetI64v(p + n, &pHeadFile->commitID); n += tGetI64v(p + n, &pHeadFile->size); n += tGetI64v(p + n, &pHeadFile->offset); - n += tGetI64v(p + n, &pHeadFile->loffset); return n; } @@ -60,6 +58,7 @@ int32_t tPutLastFile(uint8_t *p, SLastFile *pLastFile) { n += tPutI64v(p ? p + n : p, pLastFile->commitID); n += tPutI64v(p ? p + n : p, pLastFile->size); + n += tPutI64v(p ? p + n : p, pLastFile->offset); return n; } @@ -69,6 +68,7 @@ static int32_t tGetLastFile(uint8_t *p, SLastFile *pLastFile) { n += tGetI64v(p + n, &pLastFile->commitID); n += tGetI64v(p + n, &pLastFile->size); + n += tGetI64v(p + n, &pLastFile->offset); return n; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 69a4227486..324db5b111 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -502,7 +502,7 @@ _err: int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { int32_t code = 0; int64_t offset = pReader->pSet->pHeadF->offset; - int64_t size = pReader->pSet->pHeadF->loffset - offset; + int64_t size = pReader->pSet->pHeadF->size - offset; int64_t n; uint32_t delimiter; @@ -564,8 +564,8 @@ _err: int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL) { int32_t code = 0; - int64_t offset = pReader->pSet->pHeadF->loffset; - int64_t size = pReader->pSet->pHeadF->size - offset; + int64_t offset = pReader->pSet->pLastF->offset; + int64_t size = pReader->pSet->pLastF->size - offset; int64_t n; uint32_t delimiter; @@ -579,13 +579,13 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL) { if (code) goto _err; // seek - if (taosLSeekFile(pReader->pHeadFD, offset, SEEK_SET) < 0) { + if (taosLSeekFile(pReader->pLastFD, offset, SEEK_SET) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } // read - n = taosReadFile(pReader->pHeadFD, pReader->pBuf1, size); + n = taosReadFile(pReader->pLastFD, pReader->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1292,13 +1292,13 @@ _err: int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) { int32_t code = 0; - SHeadFile *pHeadFile = &pWriter->fHead; + SLastFile *pLastFile = &pWriter->fLast; int64_t size; int64_t n; // check if (taosArrayGetSize(aBlockL) == 0) { - pHeadFile->loffset = pHeadFile->size; + pLastFile->offset = pLastFile->size; goto _exit; } @@ -1324,19 +1324,19 @@ int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) { ASSERT(n + sizeof(TSCKSUM) == size); // write - n = taosWriteFile(pWriter->pHeadFD, pWriter->pBuf1, size); + n = taosWriteFile(pWriter->pLastFD, pWriter->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } // update - pHeadFile->loffset = pHeadFile->size; - pHeadFile->size += size; + pLastFile->offset = pLastFile->size; + pLastFile->size += size; _exit: tsdbTrace("vgId:%d tsdb write blockl, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), - pHeadFile->loffset, size); + pLastFile->offset, size); return code; _err: From 559286db71a002447b09bfd58d6a9c23c942381e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 Aug 2022 07:51:09 +0800 Subject: [PATCH 41/92] refactor(query): support read data from last files. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 241 ++++++++++++++++--------- 1 file changed, 157 insertions(+), 84 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 73454e2a63..48b7502e8f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -28,6 +28,11 @@ typedef struct { bool hasVal; } SIterInfo; +typedef struct { + int32_t numOfBlocks; + int32_t numOfLastBlocks; +} SBlockNumber; + typedef struct STableBlockScanInfo { uint64_t uid; TSKEY lastKey; @@ -71,10 +76,11 @@ typedef struct SBlockLoadSuppInfo { } SBlockLoadSuppInfo; typedef struct SFilesetIter { - int32_t numOfFiles; // number of total files - int32_t index; // current accessed index in the list - SArray* pFileList; // data file list + int32_t numOfFiles; // number of total files + int32_t index; // current accessed index in the list + SArray* pFileList; // data file list int32_t order; + SArray* pLastBlockList;// last block array list } SFilesetIter; typedef struct SFileDataBlockInfo { @@ -84,12 +90,14 @@ typedef struct SFileDataBlockInfo { } SFileDataBlockInfo; typedef struct SDataBlockIter { - int32_t numOfBlocks; + SBlockNumber numOfBlocks; int32_t index; - SArray* blockList; // SArray + SArray* blockList; // SArray + SArray* pLastBlockList; // last block list int32_t order; - SBlock block; // current SBlock data + SBlock block; // current SBlock data SHashObj* pTableMap; + SArray* pBlockL; // ptr to SBlockL in fileIterator } SDataBlockIter; typedef struct SFileBlockDumpInfo { @@ -132,6 +140,7 @@ struct STsdbReader { STSchema* pSchema; SDataFReader* pFileReader; SVersionRange verRange; + SArray* pLastBlock; // last block info int32_t step; STsdbReader* innerReader[2]; @@ -299,6 +308,13 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, int32 pIter->pFileList = aDFileSet; pIter->numOfFiles = numOfFileset; + pIter->pLastBlockList = taosArrayInit(4, sizeof(SBlockL)); + if (pIter->pLastBlockList == NULL) { + int32_t code = TSDB_CODE_OUT_OF_MEMORY; + tsdbError("failed to prepare the last block iterator, code:%d %s", tstrerror(code), idstr); + return code; + } + tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, idstr); return TSDB_CODE_SUCCESS; } @@ -359,7 +375,7 @@ _err: static void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, SHashObj* pTableMap) { pIter->order = order; pIter->index = -1; - pIter->numOfBlocks = -1; + pIter->numOfBlocks = (SBlockNumber){-1, -1}; if (pIter->blockList == NULL) { pIter->blockList = taosArrayInit(4, sizeof(SFileDataBlockInfo)); } else { @@ -592,10 +608,10 @@ _end: return code; } -static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, uint32_t* numOfValidTables, - int32_t* numOfBlocks) { +static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* pLastBlockIndex, + SBlockNumber * pBlockNum, SArray* pQualifiedLastBlock) { + int32_t numOfQTable= 0; size_t numOfTables = taosArrayGetSize(pIndexList); - *numOfValidTables = 0; int64_t st = taosGetTimestampUs(); size_t size = 0; @@ -640,19 +656,44 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, uint32_ return TSDB_CODE_OUT_OF_MEMORY; } - (*numOfBlocks) += 1; + pBlockNum->numOfBlocks += 1; } if (pScanInfo->pBlockList != NULL && taosArrayGetSize(pScanInfo->pBlockList) > 0) { - (*numOfValidTables) += 1; + numOfQTable += 1; } } - double el = (taosGetTimestampUs() - st) / 1000.0; - tsdbDebug("load block of %d tables completed, blocks:%d in %d tables, size:%.2f Kb, elapsed time:%.2f ms %s", - numOfTables, *numOfBlocks, *numOfValidTables, size / 1000.0, el, pReader->idStr); + size_t numOfLast = taosArrayGetSize(pLastBlockIndex); + for(int32_t i = 0; i < numOfLast; ++i) { + SBlockL* pLastBlock = taosArrayGet(pLastBlockIndex, i); + if (pLastBlock->suid != pReader->suid) { + continue; + } - pReader->cost.numOfBlocks += (*numOfBlocks); + { + // 1. time range check, todo add later +// if (pLastBlock->.minKey.ts > pReader->window.ekey || block.maxKey.ts < pReader->window.skey) { +// continue; +// } + + // 2. version range check + if (pLastBlock->minVer > pReader->verRange.maxVer || pLastBlock->maxVer < pReader->verRange.minVer) { + continue; + } + + pBlockNum->numOfLastBlocks += 1; + taosArrayPush(pQualifiedLastBlock, &i); + } + } + + int32_t total = pBlockNum->numOfLastBlocks + pBlockNum->numOfBlocks; + + double el = (taosGetTimestampUs() - st) / 1000.0; + tsdbDebug("load block of %d tables completed, blocks:%d in %d tables, lastBlock:%d, size:%.2f Kb, elapsed time:%.2f ms %s", + numOfTables, total, numOfQTable, pBlockNum->numOfLastBlocks, size / 1000.0, el, pReader->idStr); + + pReader->cost.numOfBlocks += total; pReader->cost.headFileLoadTime += el; return TSDB_CODE_SUCCESS; @@ -682,8 +723,12 @@ static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_ } static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) { - SFileDataBlockInfo* pFBlockInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index); - return pFBlockInfo; + if (pBlockIter->index >= pBlockIter->numOfBlocks.numOfBlocks) { + return NULL; + } else { + SFileDataBlockInfo* pFBlockInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index); + return pFBlockInfo; + } } static SBlock* getCurrentBlock(SDataBlockIter* pBlockIter) { return &pBlockIter->block; } @@ -771,41 +816,45 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn return TSDB_CODE_SUCCESS; } -static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockIter, - STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) { +static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockIter, SBlockData* pBlockData) { int64_t st = taosGetTimestampUs(); + double elapsedTime = 0; + int32_t code = 0; SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); - SBlock* pBlock = getCurrentBlock(pBlockIter); - - SSDataBlock* pResBlock = pReader->pResBlock; - int32_t numOfCols = blockDataGetNumOfCols(pResBlock); - - SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - SBlockIdx blockIdx = {.suid = pReader->suid, .uid = pBlockScanInfo->uid}; - int32_t code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData /*, pSupInfo->colIds, numOfCols*/); - if (code != TSDB_CODE_SUCCESS) { - goto _error; + if (pFBlock != NULL) { + SBlock* pBlock = getCurrentBlock(pBlockIter); + code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + elapsedTime = (taosGetTimestampUs() - st) / 1000.0; + + tsdbDebug("%p load file block into buffer, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 + ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s", + pReader, pBlockIter->index, pFBlock->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow, + pBlock->minVer, pBlock->maxVer, elapsedTime, pReader->idStr); + } else { + int32_t pos = pBlockIter->index - pBlockIter->numOfBlocks.numOfBlocks; + int32_t* index = taosArrayGet(pBlockIter->pLastBlockList, pos); + + SBlockL* pBlockL = taosArrayGet(pReader->status.fileIter.pLastBlockList, *index); + code = tsdbReadLastBlock(pReader->pFileReader, pBlockL, pBlockData); } - double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; pReader->cost.blockLoadTime += elapsedTime; - pDumpInfo->allDumped = false; - tsdbDebug("%p load file block into buffer, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 - ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s", - pReader, pBlockIter->index, pFBlock->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow, - pBlock->minVer, pBlock->maxVer, elapsedTime, pReader->idStr); return TSDB_CODE_SUCCESS; _error: - tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 - ", rows:%d, %s", - pReader, pBlockIter->index, pFBlock->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow, - pReader->idStr); +// tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 +// ", rows:%d, %s", +// pReader, pBlockIter->index, pFBlock->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow, +// pReader->idStr); return code; } @@ -861,10 +910,11 @@ static int32_t fileDataBlockOrderCompar(const void* pLeft, const void* pRight, v static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter) { SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); - STableBlockScanInfo* pScanInfo = taosHashGet(pBlockIter->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); - - int32_t* mapDataIndex = taosArrayGet(pScanInfo->pBlockList, pFBlock->tbBlockIdx); - tMapDataGetItemByIdx(&pScanInfo->mapData, *mapDataIndex, &pBlockIter->block, tGetBlock); + if (pFBlock != NULL) { + STableBlockScanInfo* pScanInfo = taosHashGet(pBlockIter->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); + int32_t* mapDataIndex = taosArrayGet(pScanInfo->pBlockList, pFBlock->tbBlockIdx); + tMapDataGetItemByIdx(&pScanInfo->mapData, *mapDataIndex, &pBlockIter->block, tGetBlock); + } #if 0 qDebug("check file block, table uid:%"PRIu64" index:%d offset:%"PRId64", ", pScanInfo->uid, *mapDataIndex, pBlockIter->block.aSubBlock[0].offset); @@ -873,12 +923,17 @@ static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter) { return TSDB_CODE_SUCCESS; } -static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks) { +static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, SBlockNumber* pBlockNum, SArray* pQLastBlock) { bool asc = ASCENDING_TRAVERSE(pReader->order); - pBlockIter->numOfBlocks = numOfBlocks; + pBlockIter->numOfBlocks = *pBlockNum; taosArrayClear(pBlockIter->blockList); + if (pBlockNum->numOfLastBlocks > 0) { + taosArrayDestroy(pBlockIter->pLastBlockList); + pBlockIter->pLastBlockList = pQLastBlock; + } + // access data blocks according to the offset of each block in asc/desc order. int32_t numOfTables = (int32_t)taosHashGetSize(pReader->status.pTableMap); @@ -930,20 +985,21 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte sup.numOfTables += 1; } - ASSERT(numOfBlocks == cnt); + ASSERT(pBlockNum->numOfBlocks == cnt); + int32_t total = pBlockNum->numOfLastBlocks + pBlockNum->numOfBlocks; // since there is only one table qualified, blocks are not sorted - if (sup.numOfTables == 1) { - for (int32_t i = 0; i < numOfBlocks; ++i) { + if (sup.numOfTables == 1 || pBlockNum->numOfBlocks == 0) { + for (int32_t i = 0; i < pBlockNum->numOfBlocks; ++i) { SFileDataBlockInfo blockInfo = {.uid = sup.pDataBlockInfo[0][i].uid, .tbBlockIdx = i}; taosArrayPush(pBlockIter->blockList, &blockInfo); } int64_t et = taosGetTimestampUs(); tsdbDebug("%p create blocks info struct completed for one table, %d blocks not sorted, elapsed time:%.2f ms %s", - pReader, cnt, (et - st) / 1000.0, pReader->idStr); + pReader, total, (et - st) / 1000.0, pReader->idStr); - pBlockIter->index = asc ? 0 : (numOfBlocks - 1); + pBlockIter->index = asc ? 0 : (total - 1); cleanupBlockOrderSupporter(&sup); doSetCurrentBlock(pBlockIter); return TSDB_CODE_SUCCESS; @@ -952,7 +1008,7 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables %s", pReader, cnt, sup.numOfTables, pReader->idStr); - assert(cnt <= numOfBlocks && sup.numOfTables <= numOfTables); + ASSERT(cnt <= pBlockNum->numOfBlocks && sup.numOfTables <= numOfTables); SMultiwayMergeTreeInfo* pTree = NULL; uint8_t ret = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, fileDataBlockOrderCompar); @@ -979,12 +1035,12 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte } int64_t et = taosGetTimestampUs(); - tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, cnt, (et - st) / 1000.0, + tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, total, (et - st) / 1000.0, pReader->idStr); cleanupBlockOrderSupporter(&sup); taosMemoryFree(pTree); - pBlockIter->index = asc ? 0 : (numOfBlocks - 1); + pBlockIter->index = asc ? 0 : (total - 1); doSetCurrentBlock(pBlockIter); return TSDB_CODE_SUCCESS; @@ -994,7 +1050,7 @@ static bool blockIteratorNext(SDataBlockIter* pBlockIter) { bool asc = ASCENDING_TRAVERSE(pBlockIter->order); int32_t step = asc ? 1 : -1; - if ((pBlockIter->index >= pBlockIter->numOfBlocks - 1 && asc) || (pBlockIter->index <= 0 && (!asc))) { + if ((pBlockIter->index >= pBlockIter->numOfBlocks.numOfBlocks - 1 && asc) || (pBlockIter->index <= 0 && (!asc))) { return false; } @@ -1041,7 +1097,7 @@ static int32_t findFileBlockInfoIndex(SDataBlockIter* pBlockIter, SFileDataBlock int32_t step = ASCENDING_TRAVERSE(pBlockIter->order) ? 1 : -1; int32_t index = pBlockIter->index; - while (index < pBlockIter->numOfBlocks && index >= 0) { + while (index < pBlockIter->numOfBlocks.numOfBlocks && index >= 0) { SFileDataBlockInfo* pFBlock = taosArrayGet(pBlockIter->blockList, index); if (pFBlock->uid == pFBlockInfo->uid && pFBlock->tbBlockIdx == pFBlockInfo->tbBlockIdx) { return index; @@ -1055,7 +1111,7 @@ static int32_t findFileBlockInfoIndex(SDataBlockIter* pBlockIter, SFileDataBlock } static int32_t setFileBlockActiveInBlockIter(SDataBlockIter* pBlockIter, int32_t index, int32_t step) { - if (index < 0 || index >= pBlockIter->numOfBlocks) { + if (index < 0 || index >= pBlockIter->numOfBlocks.numOfBlocks) { return -1; } @@ -1473,9 +1529,12 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI } } -static int32_t buildComposedDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) { +static int32_t buildComposedDataBlock(STsdbReader* pReader) { SSDataBlock* pResBlock = pReader->pResBlock; + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + STableBlockScanInfo* pBlockScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBlockData* pBlockData = &pReader->status.fileBlockData; int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; @@ -1696,7 +1755,7 @@ static TSDBKEY getCurrentKeyInBuf(SDataBlockIter* pBlockIter, STsdbReader* pRead return key; } -static int32_t moveToNextFile(STsdbReader* pReader, int32_t* numOfBlocks) { +static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SArray* pQLastBlock) { SReaderStatus* pStatus = &pReader->status; size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); @@ -1715,18 +1774,25 @@ static int32_t moveToNextFile(STsdbReader* pReader, int32_t* numOfBlocks) { return code; } - if (taosArrayGetSize(pIndexList) > 0) { - uint32_t numOfValidTable = 0; - code = doLoadFileBlock(pReader, pIndexList, &numOfValidTable, numOfBlocks); + code = tsdbReadBlockL(pReader->pFileReader, pStatus->fileIter.pLastBlockList); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pIndexList); + return code; + } + + if (taosArrayGetSize(pIndexList) > 0 || taosArrayGetSize(pStatus->fileIter.pLastBlockList) > 0) { + code = doLoadFileBlock(pReader, pIndexList, pStatus->fileIter.pLastBlockList, pBlockNum, pQLastBlock); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pIndexList); return code; } - if (numOfValidTable > 0) { + if (pBlockNum->numOfBlocks + pBlockNum->numOfLastBlocks > 0) { + ASSERT(taosArrayGetSize(pQLastBlock) == pBlockNum->numOfLastBlocks); break; } } + // no blocks in current file, try next files } @@ -1741,21 +1807,27 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { SDataBlockIter* pBlockIter = &pStatus->blockIter; SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); - STableBlockScanInfo* pScanInfo = taosHashGet(pStatus->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); + STableBlockScanInfo* pScanInfo = NULL; + SBlock* pBlock = NULL; + TSDBKEY key = {0}; - SBlock* pBlock = getCurrentBlock(pBlockIter); + if (pFBlock != NULL) { + pScanInfo = taosHashGet(pStatus->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); + pBlock = getCurrentBlock(pBlockIter); + key = getCurrentKeyInBuf(pBlockIter, pReader); + } - TSDBKEY key = getCurrentKeyInBuf(pBlockIter, pReader); - if (fileBlockShouldLoad(pReader, pFBlock, pBlock, pScanInfo, key)) { + if (pFBlock == NULL || fileBlockShouldLoad(pReader, pFBlock, pBlock, pScanInfo, key)) { tBlockDataReset(&pStatus->fileBlockData); tBlockDataClear(&pStatus->fileBlockData); - code = doLoadFileBlockData(pReader, pBlockIter, pScanInfo, &pStatus->fileBlockData); + + code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { return code; } // build composed data block - code = buildComposedDataBlock(pReader, pScanInfo); + code = buildComposedDataBlock(pReader); } else if (bufferDataInFileBlockGap(pReader->order, key, pBlock)) { // data in memory that are earlier than current file block // todo rows in buffer should be less than the file block in asc, greater than file block in desc @@ -1819,20 +1891,23 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) } static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBlockIter) { - int32_t numOfBlocks = 0; - int32_t code = moveToNextFile(pReader, &numOfBlocks); + SBlockNumber num = {0}; + SArray* pQLastBlock = taosArrayInit(4, sizeof(int32_t)); + + int32_t code = moveToNextFile(pReader, &num, pQLastBlock); if (code != TSDB_CODE_SUCCESS) { return code; } // all data files are consumed, try data in buffer - if (numOfBlocks == 0) { + if (num.numOfBlocks + num.numOfLastBlocks == 0) { + taosArrayDestroy(pQLastBlock); pReader->status.loadFromFile = false; return code; } // initialize the block iterator for a new fileset - code = initBlockIterator(pReader, pBlockIter, numOfBlocks); + code = initBlockIterator(pReader, pBlockIter, &num, pQLastBlock); // set the correct start position according to the query time window initBlockDumpInfo(pReader, pBlockIter); @@ -1851,13 +1926,10 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { SDataBlockIter* pBlockIter = &pReader->status.blockIter; while (1) { - SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter); - STableBlockScanInfo* pScanInfo = taosHashGet(pReader->status.pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; if (fileBlockPartiallyRead(pDumpInfo, asc)) { // file data block is partially loaded - code = buildComposedDataBlock(pReader, pScanInfo); + code = buildComposedDataBlock(pReader); } else { // current block are exhausted, try the next file block if (pDumpInfo->allDumped) { @@ -2209,7 +2281,7 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn // 3. load the neighbor block, and set it to be the currently accessed file data block tBlockDataReset(&pStatus->fileBlockData); tBlockDataClear(&pStatus->fileBlockData); - int32_t code = doLoadFileBlockData(pReader, pBlockIter, pScanInfo, &pStatus->fileBlockData); + int32_t code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2569,6 +2641,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl pCond->order = TSDB_ORDER_ASC; } + // here we only need one more row, so the capacity is set to be ONE. code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[0], 1, idstr); if (code != TSDB_CODE_SUCCESS) { goto _err; @@ -2872,7 +2945,7 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) { tBlockDataReset(&pStatus->fileBlockData); tBlockDataClear(&pStatus->fileBlockData); - int32_t code = doLoadFileBlockData(pReader, &pStatus->blockIter, pBlockScanInfo, &pStatus->fileBlockData); + int32_t code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { tBlockDataDestroy(&pStatus->fileBlockData, 1); @@ -2968,12 +3041,12 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa SDataBlockIter* pBlockIter = &pStatus->blockIter; pTableBlockInfo->numOfFiles += pStatus->fileIter.numOfFiles; - if (pBlockIter->numOfBlocks > 0) { - pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks; + if (pBlockIter->numOfBlocks.numOfBlocks > 0) { + pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks.numOfBlocks; } pTableBlockInfo->numOfTables = numOfTables; - bool hasNext = (pBlockIter->numOfBlocks > 0); + bool hasNext = (pBlockIter->numOfBlocks.numOfBlocks > 0); while (true) { if (hasNext) { @@ -3004,8 +3077,8 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa break; } - pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks; - hasNext = (pBlockIter->numOfBlocks > 0); + pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks.numOfBlocks; + hasNext = (pBlockIter->numOfBlocks.numOfBlocks > 0); } // tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %s", pReader, numOfBlocks, numOfTables, From 23a0af9d1272cbfe5efa150e84cd213c878d7089 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 17 Aug 2022 11:46:13 +0800 Subject: [PATCH 42/92] more work --- source/dnode/vnode/src/inc/tsdb.h | 16 +++------------- source/dnode/vnode/src/tsdb/tsdbCommit.c | 4 ++++ source/dnode/vnode/src/tsdb/tsdbUtil.c | 24 ++++++++++++++---------- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 7e88a884c5..88960b5706 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -302,13 +302,6 @@ size_t tsdbCacheGetCapacity(SVnode *pVnode); int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema); // structs ======================= -typedef struct { - int minFid; - int midFid; - int maxFid; - TSKEY minKey; -} SRtn; - struct STsdbFS { SDelFile *pDelFile; SArray *aDFileSet; // SArray @@ -336,19 +329,14 @@ struct SMemSkipListNode { SMemSkipListNode *forwards[0]; }; typedef struct SMemSkipList { - uint32_t seed; int64_t size; + uint32_t seed; int8_t maxLevel; int8_t level; SMemSkipListNode *pHead; SMemSkipListNode *pTail; } SMemSkipList; -struct SDelDataInfo { - tb_uid_t suid; - tb_uid_t uid; -}; - struct STbData { tb_uid_t suid; tb_uid_t uid; @@ -448,6 +436,8 @@ struct SBlockL { int64_t suid; int64_t minUid; int64_t maxUid; + TSKEY minKey; + TSKEY maxKey; int64_t minVer; int64_t maxVer; int32_t nRow; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 589bce9aa5..7374988fc9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -567,9 +567,13 @@ static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { // info blockL.suid = pBlockData->suid; blockL.nRow = pBlockData->nRow; + blockL.minKey = TSKEY_MAX; + blockL.maxKey = TSKEY_MIN; blockL.minVer = VERSION_MAX; blockL.maxVer = VERSION_MIN; for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { + blockL.minKey = TMIN(blockL.minKey, pBlockData->aTSKEY[iRow]); + blockL.maxKey = TMAX(blockL.maxKey, pBlockData->aTSKEY[iRow]); blockL.minVer = TMIN(blockL.minVer, pBlockData->aVersion[iRow]); blockL.maxVer = TMAX(blockL.maxVer, pBlockData->aVersion[iRow]); } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 897c2359fa..e890c624ec 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -284,6 +284,8 @@ int32_t tPutBlockL(uint8_t *p, void *ph) { n += tPutI64(p ? p + n : p, pBlockL->suid); n += tPutI64(p ? p + n : p, pBlockL->minUid); n += tPutI64(p ? p + n : p, pBlockL->maxUid); + n += tPutI64v(p ? p + n : p, pBlockL->minKey); + n += tPutI64v(p ? p + n : p, pBlockL->maxKey); n += tPutI64v(p ? p + n : p, pBlockL->minVer); n += tPutI64v(p ? p + n : p, pBlockL->maxVer); n += tPutI32v(p ? p + n : p, pBlockL->nRow); @@ -301,6 +303,8 @@ int32_t tGetBlockL(uint8_t *p, void *ph) { n += tGetI64(p + n, &pBlockL->suid); n += tGetI64(p + n, &pBlockL->minUid); n += tGetI64(p + n, &pBlockL->maxUid); + n += tGetI64v(p + n, &pBlockL->minKey); + n += tGetI64v(p + n, &pBlockL->maxKey); n += tGetI64v(p + n, &pBlockL->minVer); n += tGetI64v(p + n, &pBlockL->maxVer); n += tGetI32v(p + n, &pBlockL->nRow); @@ -1651,25 +1655,25 @@ void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { break; } case TSDB_DATA_TYPE_FLOAT: { - *(double*)(&pColAgg->sum) += colVal.value.f; - if (!minAssigned || *(double*)(&pColAgg->min) > colVal.value.f) { - *(double*)(&pColAgg->min) = colVal.value.f; + *(double *)(&pColAgg->sum) += colVal.value.f; + if (!minAssigned || *(double *)(&pColAgg->min) > colVal.value.f) { + *(double *)(&pColAgg->min) = colVal.value.f; minAssigned = true; } - if (!maxAssigned || *(double*)(&pColAgg->max) < colVal.value.f) { - *(double*)(&pColAgg->max) = colVal.value.f; + if (!maxAssigned || *(double *)(&pColAgg->max) < colVal.value.f) { + *(double *)(&pColAgg->max) = colVal.value.f; maxAssigned = true; } break; } case TSDB_DATA_TYPE_DOUBLE: { - *(double*)(&pColAgg->sum) += colVal.value.d; - if (!minAssigned || *(double*)(&pColAgg->min) > colVal.value.d) { - *(double*)(&pColAgg->min) = colVal.value.d; + *(double *)(&pColAgg->sum) += colVal.value.d; + if (!minAssigned || *(double *)(&pColAgg->min) > colVal.value.d) { + *(double *)(&pColAgg->min) = colVal.value.d; minAssigned = true; } - if (!maxAssigned || *(double*)(&pColAgg->max) < colVal.value.d) { - *(double*)(&pColAgg->max) = colVal.value.d; + if (!maxAssigned || *(double *)(&pColAgg->max) < colVal.value.d) { + *(double *)(&pColAgg->max) = colVal.value.d; maxAssigned = true; } break; From c6106be5da57caa6f62891ab66a8303ded35bfeb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 17 Aug 2022 15:08:44 +0800 Subject: [PATCH 43/92] refact more work --- source/dnode/vnode/src/inc/tsdb.h | 5 -- source/dnode/vnode/src/meta/metaQuery.c | 67 ++++++++++++++++++---- source/dnode/vnode/src/tsdb/tsdbCommit.c | 22 +++---- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 15 ----- 4 files changed, 65 insertions(+), 44 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 88960b5706..102b714c1e 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -342,9 +342,6 @@ struct STbData { tb_uid_t uid; TSKEY minKey; TSKEY maxKey; - int64_t minVersion; - int64_t maxVersion; - int32_t maxSkmVer; SDelData *pHead; SDelData *pTail; SMemSkipList sl; @@ -358,8 +355,6 @@ struct SMemTable { volatile int32_t nRef; TSKEY minKey; TSKEY maxKey; - int64_t minVersion; - int64_t maxVersion; int64_t nRow; int64_t nDel; struct { diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 6ff55d2f4e..3ce7826701 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -429,18 +429,65 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { } int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sver, STSchema **ppTSchema) { - int32_t code = 0; - STSchema *pTSchema = NULL; - SSkmDbKey skmDbKey = {.uid = suid ? suid : uid, .sver = sver}; + int32_t code = 0; + void *pData = NULL; int nData = 0; + SSkmDbKey skmDbKey; + if (sver <= 0) { + SMetaInfo info; + if (metaGetInfo(pMeta, suid ? suid : uid, &info) == 0) { + sver = info.skmVer; + } else { + TBC *pSkmDbC = NULL; + int c; - // query + skmDbKey.uid = suid ? suid : uid; + skmDbKey.sver = INT32_MAX; + + tdbTbcOpen(pMeta->pSkmDb, &pSkmDbC, NULL); + metaRLock(pMeta); + + if (tdbTbcMoveTo(pSkmDbC, &skmDbKey, sizeof(skmDbKey), &c) < 0) { + metaULock(pMeta); + tdbTbcClose(pSkmDbC); + code = TSDB_CODE_NOT_FOUND; + goto _exit; + } + + ASSERT(c); + + if (c < 0) { + tdbTbcMoveToPrev(pSkmDbC); + } + + const void *pKey = NULL; + int32_t nKey = 0; + tdbTbcGet(pSkmDbC, &pKey, &nKey, NULL, NULL); + + if (((SSkmDbKey *)pKey)->uid != uid) { + metaULock(pMeta); + tdbTbcClose(pSkmDbC); + code = TSDB_CODE_NOT_FOUND; + goto _exit; + } + + sver = ((SSkmDbKey *)pKey)->sver; + + metaULock(pMeta); + tdbTbcClose(pSkmDbC); + } + } + + ASSERT(sver > 0); + + skmDbKey.uid = suid ? suid : uid; + skmDbKey.sver = sver; metaRLock(pMeta); - if (tdbTbGet(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), &pData, &nData) < 0) { - code = TSDB_CODE_NOT_FOUND; + if (tdbTbGet(pMeta->pSkmDb, &skmDbKey, sizeof(SSkmDbKey), &pData, &nData) < 0) { metaULock(pMeta); - goto _err; + code = TSDB_CODE_NOT_FOUND; + goto _exit; } metaULock(pMeta); @@ -462,15 +509,13 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv SSchema *pSchema = pSchemaWrapper->pSchema + i; tdAddColToSchema(&sb, pSchema->type, pSchema->flags, pSchema->colId, pSchema->bytes); } - pTSchema = tdGetSchemaFromBuilder(&sb); + STSchema *pTSchema = tdGetSchemaFromBuilder(&sb); tdDestroyTSchemaBuilder(&sb); *ppTSchema = pTSchema; taosMemoryFree(pSchemaWrapper->pSchema); - return code; -_err: - *ppTSchema = NULL; +_exit: return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 7374988fc9..020f3b0bc6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -290,23 +290,19 @@ _err: return code; } -static int32_t tsdbCommitterUpdateTableSchema(SCommitter *pCommitter, int64_t suid, int64_t uid, int32_t sver) { +static int32_t tsdbCommitterUpdateTableSchema(SCommitter *pCommitter, int64_t suid, int64_t uid) { int32_t code = 0; - if (pCommitter->skmTable.pTSchema) { - if (pCommitter->skmTable.suid == suid) { - if (suid == 0) { - if (pCommitter->skmTable.uid == uid && sver == pCommitter->skmTable.pTSchema->version) goto _exit; - } else { - if (sver == pCommitter->skmTable.pTSchema->version) goto _exit; - } - } + if (suid) { + if (pCommitter->skmTable.suid == suid) goto _exit; + } else { + if (pCommitter->skmTable.uid == uid) goto _exit; } pCommitter->skmTable.suid = suid; pCommitter->skmTable.uid = uid; tTSchemaDestroy(pCommitter->skmTable.pTSchema); - code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmTable.pTSchema); + code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, -1, &pCommitter->skmTable.pTSchema); if (code) goto _exit; _exit: @@ -360,7 +356,7 @@ static int32_t tsdbCommitterNextLastRow(SCommitter *pCommitter) { int64_t suid = pBlockL->suid; int64_t uid = pBlockL->maxUid; - code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid, 1 /*TODO*/); + code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid); if (code) goto _exit; code = tBlockDataInit(pBlockDatal, suid, suid ? 0 : uid, pCommitter->skmTable.pTSchema); @@ -978,7 +974,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { pBlock = NULL; } - code = tsdbCommitterUpdateTableSchema(pCommitter, pTbData->suid, pTbData->uid, pTbData->maxSkmVer /*TODO*/); + code = tsdbCommitterUpdateTableSchema(pCommitter, pTbData->suid, pTbData->uid); if (code) goto _err; tMapDataReset(&pCommitter->dWriter.mBlock); @@ -1160,7 +1156,7 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { // set block data schema if need if (pBlockDataW->suid == 0 && pBlockDataW->uid == 0) { - code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid, 1 /*TOOD*/); + code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid); if (code) goto _err; code = tBlockDataInit(pBlockDataW, suid, suid ? 0 : uid, pCommitter->skmTable.pTSchema); diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 803b32517a..a6628463f8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -46,8 +46,6 @@ int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { pMemTable->nRef = 1; pMemTable->minKey = TSKEY_MAX; pMemTable->maxKey = TSKEY_MIN; - pMemTable->minVersion = VERSION_MAX; - pMemTable->maxVersion = VERSION_MIN; pMemTable->nRow = 0; pMemTable->nDel = 0; pMemTable->nTbData = 0; @@ -180,10 +178,6 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid pTbData->pTail = pDelData; } - // update the state of pMemTable and other (todo) - - pMemTable->minVersion = TMIN(pMemTable->minVersion, version); - pMemTable->maxVersion = TMAX(pMemTable->maxVersion, version); pMemTable->nDel++; if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config) && tsdbKeyCmprFn(&lastKey, &pTbData->maxKey) >= 0) { @@ -368,9 +362,6 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid pTbData->uid = uid; pTbData->minKey = TSKEY_MAX; pTbData->maxKey = TSKEY_MIN; - pTbData->minVersion = VERSION_MAX; - pTbData->maxVersion = VERSION_MIN; - pTbData->maxSkmVer = -1; pTbData->pHead = NULL; pTbData->pTail = NULL; pTbData->sl.seed = taosRand(); @@ -615,15 +606,9 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, i tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, pLastRow, pMemTable->pTsdb); } - pTbData->minVersion = TMIN(pTbData->minVersion, version); - pTbData->maxVersion = TMAX(pTbData->maxVersion, version); - pTbData->maxSkmVer = TMAX(pTbData->maxSkmVer, pMsgIter->sversion); - // SMemTable pMemTable->minKey = TMIN(pMemTable->minKey, pTbData->minKey); pMemTable->maxKey = TMAX(pMemTable->maxKey, pTbData->maxKey); - pMemTable->minVersion = TMIN(pMemTable->minVersion, pTbData->minVersion); - pMemTable->maxVersion = TMAX(pMemTable->maxVersion, pTbData->maxVersion); pMemTable->nRow += nRow; pRsp->numOfRows = nRow; From 4b47476de1e9ac2301280ab64b06d119c152efeb Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 17 Aug 2022 16:51:15 +0800 Subject: [PATCH 44/92] fix: upgrade last files reading for last/last_row caching --- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbCache.c | 221 +++++++++++++++++++++--- source/dnode/vnode/src/tsdb/tsdbUtil.c | 21 ++- 3 files changed, 222 insertions(+), 21 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 102b714c1e..6263703c01 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -123,6 +123,7 @@ int32_t tGetBlockL(uint8_t *p, void *ph); int32_t tPutBlockIdx(uint8_t *p, void *ph); int32_t tGetBlockIdx(uint8_t *p, void *ph); int32_t tCmprBlockIdx(void const *lhs, void const *rhs); +int32_t tCmprBlockL(void const *lhs, void const *rhs); // SColdata void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn); void tColDataReset(SColData *pColData); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 24d6b2f385..b515b028b3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -279,6 +279,7 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb } } + _invalidate: taosMemoryFreeClear(pTSchema); taosLRUCacheRelease(pCache, h, invalidate); @@ -404,6 +405,178 @@ _err: return code; } +typedef enum { + SFSLASTNEXTROW_FS, + SFSLASTNEXTROW_FILESET, + SFSLASTNEXTROW_BLOCKDATA, + SFSLASTNEXTROW_BLOCKROW +} SFSLASTNEXTROWSTATES; + +typedef struct { + SFSLASTNEXTROWSTATES state; // [input] + STsdb *pTsdb; // [input] + SBlockIdx *pBlockIdxExp; // [input] + STSchema *pTSchema; // [input] + int32_t nFileSet; + int32_t iFileSet; + SArray *aDFileSet; + SDataFReader *pDataFReader; + SArray *aBlockL; + SBlockL *pBlockL; + SBlockData *pBlockDataL; + SBlockData blockDataL; + int32_t nRow; + int32_t iRow; + TSDBROW row; + /* + SArray *aBlockIdx; + SBlockIdx *pBlockIdx; + SMapData blockMap; + int32_t nBlock; + int32_t iBlock; + SBlock block; + */ +} SFSLastNextRowIter; + +static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { + SFSLastNextRowIter *state = (SFSLastNextRowIter *)iter; + int32_t code = 0; + + switch (state->state) { + case SFSLASTNEXTROW_FS: + // state->aDFileSet = state->pTsdb->pFS->cState->aDFileSet; + state->nFileSet = taosArrayGetSize(state->aDFileSet); + state->iFileSet = state->nFileSet; + + state->pBlockDataL = NULL; + + case SFSLASTNEXTROW_FILESET: { + SDFileSet *pFileSet = NULL; + _next_fileset: + if (--state->iFileSet >= 0) { + pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet); + } else { + if (state->pBlockDataL) { + tBlockDataDestroy(state->pBlockDataL, 1); + state->pBlockDataL = NULL; + } + + *ppRow = NULL; + return code; + } + + code = tsdbDataFReaderOpen(&state->pDataFReader, state->pTsdb, pFileSet); + if (code) goto _err; + + if (!state->aBlockL) { + state->aBlockL = taosArrayInit(0, sizeof(SBlockIdx)); + } else { + taosArrayClear(state->aBlockL); + } + + code = tsdbReadBlockL(state->pDataFReader, state->aBlockL); + if (code) goto _err; + + // SBlockL *pBlockL = (SBlockL *)taosArrayGet(state->aBlockL, state->iBlockL); + + state->pBlockL = taosArraySearch(state->aBlockL, state->pBlockIdxExp, tCmprBlockL, TD_EQ); + if (!state->pBlockL) { + goto _next_fileset; + } + + int64_t suid = state->pBlockL->suid; + int64_t uid = state->pBlockL->maxUid; + + if (!state->pBlockDataL) { + state->pBlockDataL = &state->blockDataL; + } + code = tBlockDataInit(state->pBlockDataL, suid, suid ? 0 : uid, state->pTSchema); + if (code) goto _err; + } + case SFSLASTNEXTROW_BLOCKDATA: + code = tsdbReadLastBlock(state->pDataFReader, state->pBlockL, state->pBlockDataL); + if (code) goto _err; + + state->nRow = state->blockDataL.nRow; + state->iRow = state->nRow - 1; + + if (!state->pBlockDataL->uid) { + while (state->pBlockIdxExp->uid != state->pBlockDataL->aUid[state->iRow]) { + --state->iRow; + } + } + + state->state = SFSLASTNEXTROW_BLOCKROW; + case SFSLASTNEXTROW_BLOCKROW: + if (state->pBlockDataL->uid) { + if (state->iRow >= 0) { + state->row = tsdbRowFromBlockData(state->pBlockDataL, state->iRow); + *ppRow = &state->row; + + if (--state->iRow < 0) { + state->state = SFSLASTNEXTROW_FILESET; + } + } + } else { + if (state->iRow >= 0 && state->pBlockIdxExp->uid == state->pBlockDataL->aUid[state->iRow]) { + state->row = tsdbRowFromBlockData(state->pBlockDataL, state->iRow); + *ppRow = &state->row; + + if (--state->iRow < 0 || state->pBlockIdxExp->uid != state->pBlockDataL->aUid[state->iRow]) { + state->state = SFSLASTNEXTROW_FILESET; + } + } + } + + return code; + default: + ASSERT(0); + break; + } + +_err: + if (state->pDataFReader) { + tsdbDataFReaderClose(&state->pDataFReader); + state->pDataFReader = NULL; + } + if (state->aBlockL) { + taosArrayDestroy(state->aBlockL); + state->aBlockL = NULL; + } + if (state->pBlockDataL) { + tBlockDataDestroy(state->pBlockDataL, 1); + state->pBlockDataL = NULL; + } + + *ppRow = NULL; + + return code; +} + +int32_t clearNextRowFromFSLast(void *iter) { + SFSLastNextRowIter *state = (SFSLastNextRowIter *)iter; + int32_t code = 0; + + if (!state) { + return code; + } + + if (state->pDataFReader) { + tsdbDataFReaderClose(&state->pDataFReader); + state->pDataFReader = NULL; + } + if (state->aBlockL) { + taosArrayDestroy(state->aBlockL); + state->aBlockL = NULL; + } + if (state->pBlockDataL) { + tBlockDataDestroy(state->pBlockDataL, 1); + state->pBlockDataL = NULL; + } + + return code; +} + typedef enum SFSNEXTROWSTATES { SFSNEXTROW_FS, SFSNEXTROW_FILESET, @@ -722,18 +895,19 @@ typedef struct { SArray *pSkyline; int64_t iSkyline; - SBlockIdx idx; - SMemNextRowIter memState; - SMemNextRowIter imemState; - SFSNextRowIter fsState; - TSDBROW memRow, imemRow, fsRow; + SBlockIdx idx; + SMemNextRowIter memState; + SMemNextRowIter imemState; + SFSLastNextRowIter fsLastState; + SFSNextRowIter fsState; + TSDBROW memRow, imemRow, fsLastRow, fsRow; - TsdbNextRowState input[3]; + TsdbNextRowState input[4]; STsdbReadSnap *pReadSnap; STsdb *pTsdb; } CacheNextRowIter; -static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb) { +static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb, STSchema *pTSchema) { int code = 0; tb_uid_t suid = getTableSuidByUid(uid, pTsdb); @@ -779,6 +953,12 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->idx = (SBlockIdx){.suid = suid, .uid = uid}; + pIter->fsLastState.state = SFSNEXTROW_FS; + pIter->fsLastState.pTsdb = pTsdb; + pIter->fsLastState.aDFileSet = pIter->pReadSnap->fs.aDFileSet; + pIter->fsLastState.pBlockIdxExp = &pIter->idx; + pIter->fsLastState.pTSchema = pTSchema; + pIter->fsState.state = SFSNEXTROW_FS; pIter->fsState.pTsdb = pTsdb; pIter->fsState.aDFileSet = pIter->pReadSnap->fs.aDFileSet; @@ -786,7 +966,9 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->input[0] = (TsdbNextRowState){&pIter->memRow, true, false, &pIter->memState, getNextRowFromMem, NULL}; pIter->input[1] = (TsdbNextRowState){&pIter->imemRow, true, false, &pIter->imemState, getNextRowFromMem, NULL}; - pIter->input[2] = + pIter->input[2] = (TsdbNextRowState){&pIter->fsLastRow, false, true, &pIter->fsLastState, getNextRowFromFSLast, + clearNextRowFromFSLast}; + pIter->input[3] = (TsdbNextRowState){&pIter->fsRow, false, true, &pIter->fsState, getNextRowFromFS, clearNextRowFromFS}; if (pMem) { @@ -811,7 +993,7 @@ _err: static int32_t nextRowIterClose(CacheNextRowIter *pIter) { int code = 0; - for (int i = 0; i < 3; ++i) { + for (int i = 0; i < 4; ++i) { if (pIter->input[i].nextRowClearFn) { pIter->input[i].nextRowClearFn(pIter->input[i].iter); } @@ -823,7 +1005,6 @@ static int32_t nextRowIterClose(CacheNextRowIter *pIter) { tsdbUntakeReadSnap(pIter->pTsdb, pIter->pReadSnap); - return code; _err: return code; } @@ -832,7 +1013,7 @@ _err: static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow) { int code = 0; - for (int i = 0; i < 3; ++i) { + for (int i = 0; i < 4; ++i) { if (pIter->input[i].next && !pIter->input[i].stop) { code = pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow); if (code) goto _err; @@ -844,18 +1025,18 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow) { } } - if (pIter->input[0].stop && pIter->input[1].stop && pIter->input[2].stop) { + if (pIter->input[0].stop && pIter->input[1].stop && pIter->input[2].stop && pIter->input[3].stop) { *ppRow = NULL; return code; } - // select maxpoint(s) from mem, imem, fs - TSDBROW *max[3] = {0}; - int iMax[3] = {-1, -1, -1}; + // select maxpoint(s) from mem, imem, fs and last + TSDBROW *max[4] = {0}; + int iMax[4] = {-1, -1, -1, -1}; int nMax = 0; TSKEY maxKey = TSKEY_MIN; - for (int i = 0; i < 3; ++i) { + for (int i = 0; i < 4; ++i) { if (!pIter->input[i].stop && pIter->input[i].pRow != NULL) { TSDBKEY key = TSDBROW_KEY(pIter->input[i].pRow); @@ -873,8 +1054,8 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow) { } // delete detection - TSDBROW *merge[3] = {0}; - int iMerge[3] = {-1, -1, -1}; + TSDBROW *merge[4] = {0}; + int iMerge[4] = {-1, -1, -1, -1}; int nMerge = 0; for (int i = 0; i < nMax; ++i) { TSDBKEY maxKey = TSDBROW_KEY(max[i]); @@ -915,7 +1096,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo TSKEY lastRowTs = TSKEY_MAX; CacheNextRowIter iter = {0}; - nextRowIterOpen(&iter, uid, pTsdb); + nextRowIterOpen(&iter, uid, pTsdb, pTSchema); do { TSDBROW *pRow = NULL; @@ -1012,7 +1193,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) { TSKEY lastRowTs = TSKEY_MAX; CacheNextRowIter iter = {0}; - nextRowIterOpen(&iter, uid, pTsdb); + nextRowIterOpen(&iter, uid, pTsdb, pTSchema); do { TSDBROW *pRow = NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index e890c624ec..9fcfd9c8bd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -196,6 +196,25 @@ int32_t tCmprBlockIdx(void const *lhs, void const *rhs) { return 0; } +int32_t tCmprBlockL(void const *lhs, void const *rhs) { + SBlockIdx *lBlockIdx = (SBlockIdx *)lhs; + SBlockL *rBlockL = (SBlockL *)rhs; + + if (lBlockIdx->suid < rBlockL->suid) { + return -1; + } else if (lBlockIdx->suid > rBlockL->suid) { + return 1; + } + + if (lBlockIdx->uid < rBlockL->minUid) { + return -1; + } else if (lBlockIdx->uid > rBlockL->maxUid) { + return 1; + } + + return 0; +} + // SBlock ====================================================== void tBlockReset(SBlock *pBlock) { *pBlock = (SBlock){.minKey = TSDBKEY_MAX, .maxKey = TSDBKEY_MIN, .minVer = VERSION_MAX, .maxVer = VERSION_MIN}; @@ -1944,4 +1963,4 @@ int32_t tsdbReadAndCheck(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t _exit: return code; -} \ No newline at end of file +} From 9dc5b4cb0c532d31b65b423167eb8dd1a12a8479 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 18 Aug 2022 00:08:09 +0800 Subject: [PATCH 45/92] fix(query): support read from last file. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 570 ++++++++++++++++++++----- 1 file changed, 457 insertions(+), 113 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 96567ad009..048c64032c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -37,13 +37,14 @@ typedef struct { typedef struct STableBlockScanInfo { uint64_t uid; TSKEY lastKey; - SMapData mapData; // block info (compressed) - SArray* pBlockList; // block data index list - SIterInfo iter; // mem buffer skip list iterator - SIterInfo iiter; // imem buffer skip list iterator - SArray* delSkyline; // delete info for this table + SMapData mapData; // block info (compressed) + SArray* pBlockList; // block data index list + SIterInfo iter; // mem buffer skip list iterator + SIterInfo iiter; // imem buffer skip list iterator + SArray* delSkyline; // delete info for this table int32_t fileDelIndex; - bool iterInit; // whether to initialize the in-memory skip list iterator or not + bool iterInit; // whether to initialize the in-memory skip list iterator or not + int16_t indexInBlockL;// row position in last block } STableBlockScanInfo; typedef struct SBlockOrderWrapper { @@ -76,12 +77,27 @@ typedef struct SBlockLoadSuppInfo { char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated. } SBlockLoadSuppInfo; +typedef struct SVersionRange { + uint64_t minVer; + uint64_t maxVer; +} SVersionRange; + +typedef struct SLastBlockReader { + SArray* pBlockL; + int32_t currentBlockIndex; + SBlockData lastBlockData; + STimeWindow window; + SVersionRange verRange; + uint64_t uid; + int32_t rowIndex; +} SLastBlockReader; + typedef struct SFilesetIter { int32_t numOfFiles; // number of total files int32_t index; // current accessed index in the list SArray* pFileList; // data file list int32_t order; - SArray* pLastBlockList;// last block array list + SLastBlockReader* pLastBlockReader; // last file block reader } SFilesetIter; typedef struct SFileDataBlockInfo { @@ -91,14 +107,12 @@ typedef struct SFileDataBlockInfo { } SFileDataBlockInfo; typedef struct SDataBlockIter { - SBlockNumber numOfBlocks; + int32_t numOfBlocks; int32_t index; SArray* blockList; // SArray - SArray* pLastBlockList; // last block list int32_t order; SBlock block; // current SBlock data SHashObj* pTableMap; - SArray* pBlockL; // ptr to SBlockL in fileIterator } SDataBlockIter; typedef struct SFileBlockDumpInfo { @@ -108,11 +122,6 @@ typedef struct SFileBlockDumpInfo { bool allDumped; } SFileBlockDumpInfo; -typedef struct SVersionRange { - uint64_t minVer; - uint64_t maxVer; -} SVersionRange; - typedef struct SReaderStatus { bool loadFromFile; // check file stage SHashObj* pTableMap; // SHash @@ -142,7 +151,6 @@ struct STsdbReader { STSchema* pMemSchema; // the previous schema for in-memory data, to avoid load schema too many times SDataFReader* pFileReader; SVersionRange verRange; - SArray* pLastBlock; // last block info int32_t step; STsdbReader* innerReader[2]; @@ -154,10 +162,11 @@ static int buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, i static TSDBROW* getValidRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader); static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader, SRowMerger* pMerger); +static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, int64_t ts, SRowMerger* pMerger); static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, STsdbReader* pReader); static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid); -static int32_t doAppendRowFromBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, +static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, int32_t rowIndex); static void setComposedBlockFlag(STsdbReader* pReader, bool composed); static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order); @@ -171,6 +180,8 @@ static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdb static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions, const char* idstr, int8_t* pLevel); static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level); +static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader); +static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); static int32_t setColumnIdSlotList(STsdbReader* pReader, SSDataBlock* pBlock) { SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; @@ -191,7 +202,6 @@ static int32_t setColumnIdSlotList(STsdbReader* pReader, SSDataBlock* pBlock) { if (IS_VAR_DATA_TYPE(pCol->info.type)) { pSupInfo->buildBuf[i] = taosMemoryMalloc(pCol->info.bytes); - // tsdbInfo("-------------------%d\n", pCol->info.bytes); } } @@ -208,7 +218,7 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK } for (int32_t j = 0; j < numOfTables; ++j) { - STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid}; + STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid, .indexInBlockL = -1}; if (ASCENDING_TRAVERSE(pTsdbReader->order)) { if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReader->window.skey) { info.lastKey = pTsdbReader->window.skey; @@ -310,11 +320,15 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, int32 pIter->pFileList = aDFileSet; pIter->numOfFiles = numOfFileset; - pIter->pLastBlockList = taosArrayInit(4, sizeof(SBlockL)); - if (pIter->pLastBlockList == NULL) { - int32_t code = TSDB_CODE_OUT_OF_MEMORY; - tsdbError("failed to prepare the last block iterator, code:%d %s", tstrerror(code), idstr); - return code; + if (pIter->pLastBlockReader == NULL) { + pIter->pLastBlockReader = taosMemoryCalloc(1, sizeof(struct SLastBlockReader)); + if (pIter->pLastBlockReader == NULL) { + int32_t code = TSDB_CODE_OUT_OF_MEMORY; + tsdbError("failed to prepare the last block iterator, code:%d %s", tstrerror(code), idstr); + return code; + } + + pIter->pLastBlockReader->pBlockL = taosArrayInit(4, sizeof(SBlockL)); } tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, idstr); @@ -377,7 +391,7 @@ _err: static void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, SHashObj* pTableMap) { pIter->order = order; pIter->index = -1; - pIter->numOfBlocks = (SBlockNumber){-1, -1}; + pIter->numOfBlocks = -1; if (pIter->blockList == NULL) { pIter->blockList = taosArrayInit(4, sizeof(SFileDataBlockInfo)); } else { @@ -685,7 +699,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* } pBlockNum->numOfLastBlocks += 1; - taosArrayPush(pQualifiedLastBlock, &i); + taosArrayPush(pQualifiedLastBlock, pLastBlock); } } @@ -726,12 +740,13 @@ static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_ } static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) { - if (pBlockIter->index >= pBlockIter->numOfBlocks.numOfBlocks) { + if (taosArrayGetSize(pBlockIter->blockList) == 0) { + ASSERT(pBlockIter->numOfBlocks == taosArrayGetSize(pBlockIter->blockList)); return NULL; - } else { - SFileDataBlockInfo* pFBlockInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index); - return pFBlockInfo; } + + SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index); + return pBlockInfo; } static SBlock* getCurrentBlock(SDataBlockIter* pBlockIter) { return &pBlockIter->block; } @@ -824,13 +839,17 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI double elapsedTime = 0; int32_t code = 0; - SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - if (pFBlock != NULL) { + if (pBlockInfo != NULL) { SBlock* pBlock = getCurrentBlock(pBlockIter); code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData); if (code != TSDB_CODE_SUCCESS) { + tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 + ", rows:%d, code:%s %s", + pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow, + tstrerror(code), pReader->idStr); goto _error; } @@ -838,14 +857,41 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI tsdbDebug("%p load file block into buffer, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s", - pReader, pBlockIter->index, pFBlock->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow, + pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow, pBlock->minVer, pBlock->maxVer, elapsedTime, pReader->idStr); } else { - int32_t pos = pBlockIter->index - pBlockIter->numOfBlocks.numOfBlocks; - int32_t* index = taosArrayGet(pBlockIter->pLastBlockList, pos); +#if 0 + SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; - SBlockL* pBlockL = taosArrayGet(pReader->status.fileIter.pLastBlockList, *index); + uint64_t uid = pBlockInfo->uid; + SArray* pBlocks = pLastBlockReader->pBlockL; + + pLastBlockReader->currentBlockIndex = -1; + + // find the correct SBlockL + for(int32_t i = 0; i < taosArrayGetSize(pBlocks); ++i) { + SBlockL* pBlock = taosArrayGet(pBlocks, i); + if (pBlock->minUid >= uid && pBlock->maxUid <= uid) { + pLastBlockReader->currentBlockIndex = i; + break; + } + } + +// SBlockL* pBlockL = taosArrayGet(pLastBlockReader->pBlockL, *index); code = tsdbReadLastBlock(pReader->pFileReader, pBlockL, pBlockData); + if (code != TSDB_CODE_SUCCESS) { + tsdbDebug("%p error occurs in loading last block into buffer, last block index:%d, total:%d brange:%" PRId64 "-%" PRId64 + ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", code:%s %s", + pReader, *index, pBlockIter->numOfBlocks.numOfLastBlocks, 0, 0, pBlockL->nRow, + pBlockL->minVer, pBlockL->maxVer, tstrerror(code), pReader->idStr); + goto _error; + } + + tsdbDebug("%p load last file block into buffer, last block index:%d, total:%d brange:%" PRId64 "-%" PRId64 + ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s", + pReader, *index, pBlockIter->numOfBlocks.numOfLastBlocks, 0, 0, pBlockL->nRow, + pBlockL->minVer, pBlockL->maxVer, elapsedTime, pReader->idStr); +#endif } pReader->cost.blockLoadTime += elapsedTime; @@ -854,10 +900,6 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI return TSDB_CODE_SUCCESS; _error: -// tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 -// ", rows:%d, %s", -// pReader, pBlockIter->index, pFBlock->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow, -// pReader->idStr); return code; } @@ -926,17 +968,12 @@ static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter) { return TSDB_CODE_SUCCESS; } -static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, SBlockNumber* pBlockNum, SArray* pQLastBlock) { +static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks) { bool asc = ASCENDING_TRAVERSE(pReader->order); - pBlockIter->numOfBlocks = *pBlockNum; + pBlockIter->numOfBlocks = numOfBlocks; taosArrayClear(pBlockIter->blockList); - if (pBlockNum->numOfLastBlocks > 0) { - taosArrayDestroy(pBlockIter->pLastBlockList); - pBlockIter->pLastBlockList = pQLastBlock; - } - // access data blocks according to the offset of each block in asc/desc order. int32_t numOfTables = (int32_t)taosHashGetSize(pReader->status.pTableMap); @@ -988,21 +1025,20 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte sup.numOfTables += 1; } - ASSERT(pBlockNum->numOfBlocks == cnt); - int32_t total = pBlockNum->numOfLastBlocks + pBlockNum->numOfBlocks; + ASSERT(numOfBlocks == cnt); // since there is only one table qualified, blocks are not sorted - if (sup.numOfTables == 1 || pBlockNum->numOfBlocks == 0) { - for (int32_t i = 0; i < pBlockNum->numOfBlocks; ++i) { + if (sup.numOfTables == 1) { + for (int32_t i = 0; i < numOfBlocks; ++i) { SFileDataBlockInfo blockInfo = {.uid = sup.pDataBlockInfo[0][i].uid, .tbBlockIdx = i}; taosArrayPush(pBlockIter->blockList, &blockInfo); } int64_t et = taosGetTimestampUs(); tsdbDebug("%p create blocks info struct completed for one table, %d blocks not sorted, elapsed time:%.2f ms %s", - pReader, total, (et - st) / 1000.0, pReader->idStr); + pReader, numOfBlocks, (et - st) / 1000.0, pReader->idStr); - pBlockIter->index = asc ? 0 : (total - 1); + pBlockIter->index = asc ? 0 : (numOfBlocks - 1); cleanupBlockOrderSupporter(&sup); doSetCurrentBlock(pBlockIter); return TSDB_CODE_SUCCESS; @@ -1011,7 +1047,7 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables %s", pReader, cnt, sup.numOfTables, pReader->idStr); - ASSERT(cnt <= pBlockNum->numOfBlocks && sup.numOfTables <= numOfTables); + ASSERT(cnt <= numOfBlocks && sup.numOfTables <= numOfTables); SMultiwayMergeTreeInfo* pTree = NULL; uint8_t ret = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, fileDataBlockOrderCompar); @@ -1038,12 +1074,12 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte } int64_t et = taosGetTimestampUs(); - tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, total, (et - st) / 1000.0, + tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, numOfBlocks, (et - st) / 1000.0, pReader->idStr); cleanupBlockOrderSupporter(&sup); taosMemoryFree(pTree); - pBlockIter->index = asc ? 0 : (total - 1); + pBlockIter->index = asc ? 0 : (numOfBlocks - 1); doSetCurrentBlock(pBlockIter); return TSDB_CODE_SUCCESS; @@ -1053,7 +1089,7 @@ static bool blockIteratorNext(SDataBlockIter* pBlockIter) { bool asc = ASCENDING_TRAVERSE(pBlockIter->order); int32_t step = asc ? 1 : -1; - if ((pBlockIter->index >= pBlockIter->numOfBlocks.numOfBlocks - 1 && asc) || (pBlockIter->index <= 0 && (!asc))) { + if ((pBlockIter->index >= pBlockIter->numOfBlocks - 1 && asc) || (pBlockIter->index <= 0 && (!asc))) { return false; } @@ -1100,7 +1136,7 @@ static int32_t findFileBlockInfoIndex(SDataBlockIter* pBlockIter, SFileDataBlock int32_t step = ASCENDING_TRAVERSE(pBlockIter->order) ? 1 : -1; int32_t index = pBlockIter->index; - while (index < pBlockIter->numOfBlocks.numOfBlocks && index >= 0) { + while (index < pBlockIter->numOfBlocks && index >= 0) { SFileDataBlockInfo* pFBlock = taosArrayGet(pBlockIter->blockList, index); if (pFBlock->uid == pFBlockInfo->uid && pFBlock->tbBlockIdx == pFBlockInfo->tbBlockIdx) { return index; @@ -1114,7 +1150,7 @@ static int32_t findFileBlockInfoIndex(SDataBlockIter* pBlockIter, SFileDataBlock } static int32_t setFileBlockActiveInBlockIter(SDataBlockIter* pBlockIter, int32_t index, int32_t step) { - if (index < 0 || index >= pBlockIter->numOfBlocks.numOfBlocks) { + if (index < 0 || index >= pBlockIter->numOfBlocks) { return -1; } @@ -1224,7 +1260,7 @@ static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBl // 4. output buffer should be large enough to hold all rows in current block // 5. delete info should not overlap with current block data static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBlock, SBlock* pBlock, - STableBlockScanInfo* pScanInfo, TSDBKEY key) { + STableBlockScanInfo* pScanInfo, TSDBKEY key, SLastBlockReader* pLastBlockReader) { int32_t neighborIndex = 0; SBlock* pNeighbor = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &neighborIndex, pReader->order); @@ -1239,8 +1275,17 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBloc bool hasDup = (pBlock->nSubBlock == 1) ? pBlock->hasDup : true; bool overlapWithDel = overlapWithDelSkyline(pScanInfo, pBlock, pReader->order); + // todo here we need to each key in the last files to identify if it is really overlapped with last block + bool overlapWithlastBlock = false; + if (hasDataInLastBlock(pLastBlockReader)) { + SBlockL *pBlockL = taosArrayGet(pLastBlockReader->pBlockL, pLastBlockReader->currentBlockIndex); +// int64_t ts = getCurrentKeyInLastBlock(pLastBlockReader); + overlapWithlastBlock = !(pBlock->maxKey.ts < pBlockL->minKey || pBlock->minKey.ts > pBlockL->maxKey); + } + return (overlapWithNeighbor || hasDup || dataBlockPartiallyRequired(&pReader->window, &pReader->verRange, pBlock) || - keyOverlapFileBlock(key, pBlock, &pReader->verRange) || (pBlock->nRow > pReader->capacity) || overlapWithDel); + keyOverlapFileBlock(key, pBlock, &pReader->verRange) || (pBlock->nRow > pReader->capacity) || + overlapWithDel || overlapWithlastBlock); } static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, int64_t endKey) { @@ -1279,7 +1324,7 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB int64_t nextKey = pBlockData->aTSKEY[pDumpInfo->rowIndex + step]; if (nextKey != key) { // merge is not needed - doAppendRowFromBlock(pReader->pResBlock, pReader, pBlockData, pDumpInfo->rowIndex); + doAppendRowFromFileBlock(pReader->pResBlock, pReader, pBlockData, pDumpInfo->rowIndex); pDumpInfo->rowIndex += step; return true; } @@ -1386,12 +1431,11 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return TSDB_CODE_SUCCESS; } -static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) { +static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) { SRowMerger merge = {0}; STSRow* pTSRow = NULL; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - SBlockData* pBlockData = &pReader->status.fileBlockData; SArray* pDelList = pBlockScanInfo->delSkyline; TSDBROW* pRow = getValidRow(&pBlockScanInfo->iter, pDelList, pReader); @@ -1532,6 +1576,14 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { + // it is an multi-table data block + if (pBlockData->aUid != NULL) { + uint64_t uid = pBlockData->aUid[pDumpInfo->rowIndex]; + if (uid != pBlockScanInfo->uid) { // move to next row + return false; + } + } + // check for version and time range int64_t ver = pBlockData->aVersion[pDumpInfo->rowIndex]; if (ver > pReader->verRange.maxVer || ver < pReader->verRange.minVer) { @@ -1553,16 +1605,87 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } -static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) { - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - SBlockData* pBlockData = &pReader->status.fileBlockData; +static void initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid, STimeWindow* pWin, SVersionRange* pVerRange, + int16_t startPos) { + pLastBlockReader->uid = uid; + pLastBlockReader->window = *pWin; + pLastBlockReader->verRange = *pVerRange; + pLastBlockReader->rowIndex = startPos; +} + +static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { + if (pLastBlockReader->rowIndex >= pLastBlockReader->lastBlockData.nRow) { + return false; + } + + pLastBlockReader->rowIndex += 1; + + SBlockData* pBlockData = &pLastBlockReader->lastBlockData; + for(int32_t i = pLastBlockReader->rowIndex; i < pBlockData->nRow; ++i) { + if (pBlockData->aUid[i] != pLastBlockReader->uid) { + continue; + } + + if (pBlockData->aTSKEY[i] < pLastBlockReader->window.skey) { + continue; + } + + if (pBlockData->aVersion[i] < pLastBlockReader->verRange.minVer) { + continue; + } + + // no data any more + if (pBlockData->aTSKEY[i] > pLastBlockReader->window.ekey) { + return false; + } + + if (pBlockData->aVersion[i] > pLastBlockReader->verRange.maxVer) { + return false; + } + + pLastBlockReader->rowIndex = i; + return true; + } + + pLastBlockReader->rowIndex = pBlockData->nRow; + return false; +} + +static int32_t saveCurrentState(SLastBlockReader* pLastBlockReader) { + return pLastBlockReader->rowIndex; +} + +static void restoreState(SLastBlockReader* pLastBlockReader, int32_t state) { + pLastBlockReader->rowIndex = state; +} + +static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { + SBlockData* pBlockData = &pLastBlockReader->lastBlockData; + return pBlockData->aTSKEY[pLastBlockReader->rowIndex]; +} + +// todo handle desc order +static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { + if (pLastBlockReader->rowIndex >= pLastBlockReader->lastBlockData.nRow) { + return false; + } + + return true; +} + +static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader *pLastBlockReader) { + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + + int64_t key = INT64_MIN; + if (pBlockData->nRow > 0) { + key = pBlockData->aTSKEY[pDumpInfo->rowIndex]; + } - int64_t key = pBlockData->aTSKEY[pDumpInfo->rowIndex]; TSDBROW* pRow = getValidRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); TSDBROW* piRow = getValidRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); if (pBlockScanInfo->iter.hasVal && pBlockScanInfo->iiter.hasVal) { - return doMergeThreeLevelRows(pReader, pBlockScanInfo); + return doMergeThreeLevelRows(pReader, pBlockScanInfo, pBlockData); } else { // imem + file if (pBlockScanInfo->iiter.hasVal) { @@ -1574,18 +1697,74 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI return doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, key); } - // imem & mem are all empty, only file exist - if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { - return TSDB_CODE_SUCCESS; - } else { + if (pBlockData->nRow > 0) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + // row in last file block + int64_t ts = getCurrentKeyInLastBlock(pLastBlockReader); + if (ts < key) { // save rows in last block + SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; + int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); + + STSRow* pTSRow = NULL; + SRowMerger merge = {0}; + + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, pLastBlockReader->rowIndex); + + tRowMergerInit(&merge, &fRow1, pReader->pSchema); + doMergeRowsInLastBlock(pLastBlockReader, tsLastBlock, &merge); + tRowMergerGetRow(&merge, &pTSRow); + + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + + taosMemoryFree(pTSRow); + tRowMergerClear(&merge); + return TSDB_CODE_SUCCESS; + } else if (ts == key) { + STSRow* pTSRow = NULL; + SRowMerger merge = {0}; + + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + doMergeRowsInLastBlock(pLastBlockReader, ts, &merge); + + tRowMergerGetRow(&merge, &pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + + taosMemoryFree(pTSRow); + tRowMergerClear(&merge); + return TSDB_CODE_SUCCESS; + } else { // ts > key, asc; todo handle desc + // imem & mem are all empty, only file exist + if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { + return TSDB_CODE_SUCCESS; + } else { + STSRow* pTSRow = NULL; + SRowMerger merge = {0}; + + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + tRowMergerGetRow(&merge, &pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + + taosMemoryFree(pTSRow); + tRowMergerClear(&merge); + return TSDB_CODE_SUCCESS; + } + } + } else { // only last block exists + SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; + int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); + STSRow* pTSRow = NULL; SRowMerger merge = {0}; + TSDBROW fRow = tsdbRowFromBlockData(pLastBlockData, pLastBlockReader->rowIndex); + tRowMergerInit(&merge, &fRow, pReader->pSchema); - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + doMergeRowsInLastBlock(pLastBlockReader, tsLastBlock, &merge); tRowMergerGetRow(&merge, &pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); taosMemoryFree(pTSRow); @@ -1599,25 +1778,34 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { SSDataBlock* pResBlock = pReader->pResBlock; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); - STableBlockScanInfo* pBlockScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + + STableBlockScanInfo* pBlockScanInfo = NULL; + if (pBlockInfo != NULL) { + pBlockScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + } else { + pBlockScanInfo = pReader->status.pTableIter; + } + + SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + + int16_t startIndex = pBlockInfo != NULL? pBlockScanInfo->indexInBlockL:-1; + initLastBlockReader(pLastBlockReader, pBlockScanInfo->uid, &pReader->window, &pReader->verRange, startIndex); + + bool has = nextRowInLastBlock(pLastBlockReader); // todo handle if no data in last block SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBlockData* pBlockData = &pReader->status.fileBlockData; int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; - int32_t numOfSub = 1; - int64_t st = taosGetTimestampUs(); while (1) { // todo check the validate of row in file block { - if (!isValidFileBlockRow(pBlockData, pDumpInfo, pBlockScanInfo, pReader)) { + if (pBlockData->nRow > 0 && !isValidFileBlockRow(pBlockData, pDumpInfo, pBlockScanInfo, pReader)) { pDumpInfo->rowIndex += step; SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); - numOfSub = pBlock->nSubBlock; - if (pDumpInfo->rowIndex >= pBlock->nRow || pDumpInfo->rowIndex < 0) { setBlockAllDumped(pDumpInfo, pBlock, pReader->order); break; @@ -1625,13 +1813,17 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { continue; } + + if (!hasDataInLastBlock(pLastBlockReader)) { + break; + } } - buildComposedDataBlockImpl(pReader, pBlockScanInfo); + buildComposedDataBlockImpl(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); // currently loaded file data block is consumed - if (pDumpInfo->rowIndex >= pBlock->nRow || pDumpInfo->rowIndex < 0) { + if (pBlockData->nRow > 0 && (pDumpInfo->rowIndex >= pBlock->nRow || pDumpInfo->rowIndex < 0)) { setBlockAllDumped(pDumpInfo, pBlock, pReader->order); break; } @@ -1647,9 +1839,9 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { setComposedBlockFlag(pReader, true); int64_t et = taosGetTimestampUs(); - tsdbDebug("%p uid:%" PRIu64 ", composed data block created, subBlock:%d, brange:%" PRIu64 "-%" PRIu64 + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%d, elapsed time:%.2f ms %s", - pReader, pBlockScanInfo->uid, numOfSub, pResBlock->info.window.skey, pResBlock->info.window.ekey, + pReader, pBlockScanInfo->uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, (et - st) / 1000.0, pReader->idStr); return TSDB_CODE_SUCCESS; @@ -1825,7 +2017,7 @@ static TSDBKEY getCurrentKeyInBuf(SDataBlockIter* pBlockIter, STsdbReader* pRead return key; } -static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SArray* pQLastBlock) { +static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { SReaderStatus* pStatus = &pReader->status; size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); @@ -1844,21 +2036,29 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr return code; } - code = tsdbReadBlockL(pReader->pFileReader, pStatus->fileIter.pLastBlockList); + SArray* pLastBlocks = pStatus->fileIter.pLastBlockReader->pBlockL; + code = tsdbReadBlockL(pReader->pFileReader, pLastBlocks); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pIndexList); return code; } - if (taosArrayGetSize(pIndexList) > 0 || taosArrayGetSize(pStatus->fileIter.pLastBlockList) > 0) { - code = doLoadFileBlock(pReader, pIndexList, pStatus->fileIter.pLastBlockList, pBlockNum, pQLastBlock); + if (taosArrayGetSize(pIndexList) > 0 || taosArrayGetSize(pLastBlocks) > 0) { + SArray* pQLastBlock = taosArrayInit(4, sizeof(SBlockL)); + + code = doLoadFileBlock(pReader, pIndexList, pLastBlocks, pBlockNum, pQLastBlock); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pIndexList); + taosArrayDestroy(pQLastBlock); return code; } if (pBlockNum->numOfBlocks + pBlockNum->numOfLastBlocks > 0) { ASSERT(taosArrayGetSize(pQLastBlock) == pBlockNum->numOfLastBlocks); + taosArrayClear(pLastBlocks); + taosArrayAddAll(pLastBlocks, pQLastBlock); + + taosArrayDestroy(pQLastBlock); break; } } @@ -1870,30 +2070,112 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr return TSDB_CODE_SUCCESS; } +static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, uint64_t uid, STsdbReader* pReader) { + SArray* pBlocks = pLastBlockReader->pBlockL; + SBlockL* pBlock = NULL; + + pLastBlockReader->currentBlockIndex = -1; + + // find the correct SBlockL + for (int32_t i = 0; i < taosArrayGetSize(pBlocks); ++i) { + SBlockL* p = taosArrayGet(pBlocks, i); + if (p->minUid <= uid && p->maxUid >= uid) { + pLastBlockReader->currentBlockIndex = i; + pBlock = p; + break; + } + } + + if (pLastBlockReader->currentBlockIndex == -1) { + return TSDB_CODE_SUCCESS; + } + + int32_t code = tBlockDataCreate(&pLastBlockReader->lastBlockData); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = tBlockDataInit(&pLastBlockReader->lastBlockData, pReader->suid, pReader->suid ? 0 : uid, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + //todo add log + return code; + } + + code = tsdbReadLastBlock(pReader->pFileReader, pBlock, &pLastBlockReader->lastBlockData); + if (code != TSDB_CODE_SUCCESS) { + // tsdbDebug("%p error occurs in loading last block into buffer, last block index:%d, total:%d brange:%" PRId64 "-%" PRId64 + // ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", code:%s %s", + // pReader, *index, pBlockIter->numOfBlocks.numOfLastBlocks, 0, 0, pBlock->nRow, + // pBlock->minVer, pBlock->maxVer, tstrerror(code), pReader->idStr); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { + SReaderStatus* pStatus = &pReader->status; + SLastBlockReader* pLastBlockReader = pStatus->fileIter.pLastBlockReader; + + while(1) { + if (pStatus->pTableIter == NULL) { + pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, NULL); + if (pStatus->pTableIter == NULL) { + return TSDB_CODE_SUCCESS; + } + } else { // let's try next table + pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); + if (pStatus->pTableIter == NULL) { + return TSDB_CODE_SUCCESS; + } + } + + // find the last block that contain the specified block uid + return doLoadRelatedLastBlock(pLastBlockReader, pStatus->pTableIter->uid, pReader); + + //todo check for all empty table + } +} + static int32_t doBuildDataBlock(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; - SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); - STableBlockScanInfo* pScanInfo = NULL; SBlock* pBlock = NULL; TSDBKEY key = {0}; + STableBlockScanInfo* pScanInfo = NULL; + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; - if (pFBlock != NULL) { - pScanInfo = taosHashGet(pStatus->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); + if (pBlockInfo != NULL) { + pScanInfo = taosHashGet(pStatus->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); pBlock = getCurrentBlock(pBlockIter); key = getCurrentKeyInBuf(pBlockIter, pReader); + + code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo->uid, pReader); + if (code != TSDB_CODE_SUCCESS) { + // todo handle error + } + + initLastBlockReader(pLastBlockReader, pBlockInfo->uid, &pReader->window, &pReader->verRange, pScanInfo->indexInBlockL); + bool hasData = nextRowInLastBlock(pLastBlockReader); + } else { + ASSERT(pBlockIter->numOfBlocks == 0); } - if (pFBlock == NULL || fileBlockShouldLoad(pReader, pFBlock, pBlock, pScanInfo, key)) { - tBlockDataReset(&pStatus->fileBlockData); - tBlockDataClear(&pStatus->fileBlockData); + if (pBlockInfo == NULL || fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, key, pLastBlockReader)) { + if (pBlockInfo != NULL) { + tBlockDataReset(&pStatus->fileBlockData); + code = tBlockDataInit(&pStatus->fileBlockData, pReader->suid, pScanInfo->uid, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + //todo + } - code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData); - if (code != TSDB_CODE_SUCCESS) { - return code; + code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } // build composed data block @@ -1962,22 +2244,24 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBlockIter) { SBlockNumber num = {0}; - SArray* pQLastBlock = taosArrayInit(4, sizeof(int32_t)); - int32_t code = moveToNextFile(pReader, &num, pQLastBlock); + int32_t code = moveToNextFile(pReader, &num); if (code != TSDB_CODE_SUCCESS) { return code; } // all data files are consumed, try data in buffer if (num.numOfBlocks + num.numOfLastBlocks == 0) { - taosArrayDestroy(pQLastBlock); pReader->status.loadFromFile = false; return code; } // initialize the block iterator for a new fileset - code = initBlockIterator(pReader, pBlockIter, &num, pQLastBlock); + if (num.numOfBlocks > 0) { + code = initBlockIterator(pReader, pBlockIter, num.numOfBlocks); + } else { + pBlockIter->numOfBlocks = 0; + } // set the correct start position according to the query time window initBlockDumpInfo(pReader, pBlockIter); @@ -1995,6 +2279,38 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { SDataBlockIter* pBlockIter = &pReader->status.blockIter; + if (pBlockIter->numOfBlocks == 0) { + _begin: + code = doLoadLastBlockSequentially(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + // all data blocks are check in last file, now let's try the next file + if (pReader->status.pTableIter == NULL) { + code = initForFirstBlockInFile(pReader, pBlockIter); + + // error happens or all the data files are completely checked + if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) { + return code; + } + + // this file does not have blocks, let's start check the last block file + if (pBlockIter->numOfBlocks == 0) { + goto _begin; + } + } + + code = doBuildDataBlock(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (pReader->pResBlock->info.rows > 0) { + return TSDB_CODE_SUCCESS; + } + } + while (1) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; @@ -2007,17 +2323,27 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { bool hasNext = blockIteratorNext(&pReader->status.blockIter); if (hasNext) { // check for the next block in the block accessed order list initBlockDumpInfo(pReader, pBlockIter); - } else { // data blocks in current file are exhausted, let's try the next file now + } else if (taosArrayGetSize(pReader->status.fileIter.pLastBlockReader->pBlockL) > 0) { // data blocks in current file are exhausted, let's try the next file now + // todo dump all data in last block if exists. + pBlockIter->numOfBlocks = 0; + taosArrayClear(pBlockIter->blockList); + tBlockDataReset(&pReader->status.fileBlockData); + goto _begin; + } else { code = initForFirstBlockInFile(pReader, pBlockIter); // error happens or all the data files are completely checked if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) { return code; } + + // this file does not have blocks, let's start check the last block file + if (pBlockIter->numOfBlocks == 0) { + goto _begin; + } } } - // current block is not loaded yet, or data in buffer may overlap with the file block. code = doBuildDataBlock(pReader); } @@ -2336,7 +2662,6 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn // 3. load the neighbor block, and set it to be the currently accessed file data block tBlockDataReset(&pStatus->fileBlockData); - tBlockDataClear(&pStatus->fileBlockData); int32_t code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2386,6 +2711,21 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc return TSDB_CODE_SUCCESS; } +// todo support desc order +int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, int64_t ts, SRowMerger* pMerger) { + while(nextRowInLastBlock(pLastBlockReader)) { + int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); + if (next1 == ts) { + TSDBROW fRow1 = tsdbRowFromBlockData(&pLastBlockReader->lastBlockData, pLastBlockReader->rowIndex); + tRowMerge(pMerger, &fRow1); + } else { + break; + } + } + + return TSDB_CODE_SUCCESS; +} + void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, STsdbReader* pReader, bool* freeTSRow) { TSDBROW* pNextRow = NULL; @@ -2558,7 +2898,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* return TSDB_CODE_SUCCESS; } -int32_t doAppendRowFromBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, int32_t rowIndex) { +int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, int32_t rowIndex) { int32_t i = 0, j = 0; int32_t outputRowIndex = pResBlock->info.rows; @@ -3021,8 +3361,12 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) { STableBlockScanInfo* pBlockScanInfo = taosHashGet(pStatus->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); tBlockDataReset(&pStatus->fileBlockData); - tBlockDataClear(&pStatus->fileBlockData); - int32_t code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData); + int32_t code = tBlockDataInit(&pStatus->fileBlockData, pReader->suid, pBlockScanInfo->uid, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + //todo + } + + code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { tBlockDataDestroy(&pStatus->fileBlockData, 1); @@ -3118,12 +3462,12 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa SDataBlockIter* pBlockIter = &pStatus->blockIter; pTableBlockInfo->numOfFiles += pStatus->fileIter.numOfFiles; - if (pBlockIter->numOfBlocks.numOfBlocks > 0) { - pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks.numOfBlocks; + if (pBlockIter->numOfBlocks > 0) { + pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks; } pTableBlockInfo->numOfTables = numOfTables; - bool hasNext = (pBlockIter->numOfBlocks.numOfBlocks > 0); + bool hasNext = (pBlockIter->numOfBlocks > 0); while (true) { if (hasNext) { @@ -3154,8 +3498,8 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa break; } - pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks.numOfBlocks; - hasNext = (pBlockIter->numOfBlocks.numOfBlocks > 0); + pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks; + hasNext = (pBlockIter->numOfBlocks > 0); } // tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %s", pReader, numOfBlocks, numOfTables, From 418810fca5695fdf2403036ca5d12353ec050401 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Aug 2022 10:09:31 +0800 Subject: [PATCH 46/92] optimize bitmap --- include/common/tdataformat.h | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 3679b3773b..93a693cea9 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -38,22 +38,22 @@ typedef struct STagVal STagVal; typedef struct STag STag; // bitmap -#define N1(n) ((1 << (n)) - 1) -#define BIT1_SIZE(n) (((n)-1) / 8 + 1) -#define BIT2_SIZE(n) (((n)-1) / 4 + 1) -#define SET_BIT1(p, i, v) \ - do { \ - (p)[(i) / 8] &= N1((i) % 8); \ - (p)[(i) / 8] |= (((uint8_t)(v)) << (((i) % 8))); \ - } while (0) +const static uint8_t BIT1_MAP[8][2] = { + {0b00000000, 0b00000001}, {0b00000001, 0b00000011}, {0b00000011, 0b00000111}, {0b00000111, 0b00001111}, + {0b00001111, 0b00011111}, {0b00011111, 0b00111111}, {0b00111111, 0b01111111}, {0b01111111, 0b11111111}, +}; -#define GET_BIT1(p, i) (((p)[(i) / 8] >> ((i) % 8)) & ((uint8_t)1)) -#define SET_BIT2(p, i, v) \ - do { \ - p[(i) / 4] &= N1((i) % 4 * 2); \ - (p)[(i) / 4] |= (((uint8_t)(v)) << (((i) % 4) * 2)); \ - } while (0) -#define GET_BIT2(p, i) (((p)[(i) / 4] >> (((i) % 4) * 2)) & ((uint8_t)3)) +const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, + {0b00000011, 0b00000111, 0b00001011, 2}, + {0b00001111, 0b00011111, 0b00101111, 4}, + {0b00111111, 0b01111111, 0b10111111, 6}}; + +#define BIT1_SIZE(n) (((n)-1) / 8 + 1) +#define BIT2_SIZE(n) (((n)-1) / 4 + 1) +#define SET_BIT1(p, i, v) ((p)[(i) / 8] &= BIT1_MAP[(i) % 8][v]) +#define GET_BIT1(p, i) (((p)[(i) / 8] >> ((i) % 8)) & ((uint8_t)1)) +#define SET_BIT2(p, i, v) ((p)[(i) / 4] &= BIT2_MAP[(i) % 4][v]) +#define GET_BIT2(p, i) (((p)[(i) / 4] >> BIT2_MAP[(i) % 4][3]) & ((uint8_t)3)) // STSchema int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); @@ -171,7 +171,7 @@ struct SColVal { #pragma pack(push, 1) struct STagVal { -// char colName[TSDB_COL_NAME_LEN]; // only used for tmq_get_meta + // char colName[TSDB_COL_NAME_LEN]; // only used for tmq_get_meta union { int16_t cid; char *pKey; From 3276b62aba782dfea13a43b941051854894986b8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Aug 2022 10:29:49 +0800 Subject: [PATCH 47/92] more --- include/common/tdataformat.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 93a693cea9..bc29f3b59c 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -48,12 +48,12 @@ const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, {0b00001111, 0b00011111, 0b00101111, 4}, {0b00111111, 0b01111111, 0b10111111, 6}}; -#define BIT1_SIZE(n) (((n)-1) / 8 + 1) -#define BIT2_SIZE(n) (((n)-1) / 4 + 1) -#define SET_BIT1(p, i, v) ((p)[(i) / 8] &= BIT1_MAP[(i) % 8][v]) -#define GET_BIT1(p, i) (((p)[(i) / 8] >> ((i) % 8)) & ((uint8_t)1)) -#define SET_BIT2(p, i, v) ((p)[(i) / 4] &= BIT2_MAP[(i) % 4][v]) -#define GET_BIT2(p, i) (((p)[(i) / 4] >> BIT2_MAP[(i) % 4][3]) & ((uint8_t)3)) +#define BIT1_SIZE(n) (((n)-1) >> 3 + 1) +#define BIT2_SIZE(n) (((n)-1) >> 2 + 1) +#define SET_BIT1(p, i, v) ((p)[(i) >> 3] &= BIT1_MAP[(i)&7][v]) +#define GET_BIT1(p, i) (((p)[(i) >> 3] >> ((i)&7)) & ((uint8_t)1)) +#define SET_BIT2(p, i, v) ((p)[(i) >> 2] &= BIT2_MAP[(i)&3][v]) +#define GET_BIT2(p, i) (((p)[(i) >> 2] >> BIT2_MAP[(i)&3][3]) & ((uint8_t)3)) // STSchema int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); From 1ecf660c575939968e86954770a12d8e4d11aa66 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 18 Aug 2022 10:45:52 +0800 Subject: [PATCH 48/92] other:merge 3.0 --- source/client/CMakeLists.txt | 4 +-- source/client/test/clientTests.cpp | 48 ++++++++++++++---------------- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index f52edbe71f..95e86e7629 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -46,6 +46,6 @@ target_link_libraries( PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom ) -if(${BUILD_TEST}) +#if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) -endif(${BUILD_TEST}) \ No newline at end of file +#endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index ec270889e2..4ea5443678 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -123,7 +123,7 @@ void createNewTable(TAOS* pConn, int32_t index) { } taos_free_result(pRes); - for(int32_t i = 0; i < 100000; i += 20) { + for(int32_t i = 0; i < 3280; i += 20) { char sql[1024] = {0}; sprintf(sql, "insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" @@ -679,30 +679,28 @@ TEST(testCase, projection_query_tables) { TAOS_RES* pRes = taos_query(pConn, "use abc1"); taos_free_result(pRes); - pRes = taos_query(pConn, "explain verbose true select _wstart,count(*),a from st1 partition by a interval(1s)"); - printResult(pRes); -// pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create stable st2 (ts timestamp, k int) tags(a int)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table tu using st1 tags(1)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// for(int32_t i = 0; i < 1; ++i) { -// printf("create table :%d\n", i); -// createNewTable(pConn, i); -// } + pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create stable st2 (ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tu using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + for(int32_t i = 0; i < 2; ++i) { + printf("create table :%d\n", i); + createNewTable(pConn, i); + } // // pRes = taos_query(pConn, "select * from tu"); // if (taos_errno(pRes) != 0) { From e70cfbbd002eb0fae6e68011c901e08802b26675 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 18 Aug 2022 10:58:39 +0800 Subject: [PATCH 49/92] fix(query): add jmp macro define. --- source/libs/executor/src/executorimpl.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d15dc99122..6134e9eafa 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -76,6 +76,12 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { #define realloc u_realloc #endif +#define T_LONG_JMP(_obj, _c) \ + do { \ + assert((_c) != -1); \ + longjmp((_obj), (_c)); \ + } while (0); + #define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) #define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->interval.interval > 0) From b4c2dc137ce36e310a14d240f25798e718a6aa24 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Aug 2022 11:15:31 +0800 Subject: [PATCH 50/92] fix coredump --- include/common/tdataformat.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index bc29f3b59c..e94edfcb1e 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -48,8 +48,8 @@ const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, {0b00001111, 0b00011111, 0b00101111, 4}, {0b00111111, 0b01111111, 0b10111111, 6}}; -#define BIT1_SIZE(n) (((n)-1) >> 3 + 1) -#define BIT2_SIZE(n) (((n)-1) >> 2 + 1) +#define BIT1_SIZE(n) ((((n)-1) >> 3) + 1) +#define BIT2_SIZE(n) ((((n)-1) >> 2) + 1) #define SET_BIT1(p, i, v) ((p)[(i) >> 3] &= BIT1_MAP[(i)&7][v]) #define GET_BIT1(p, i) (((p)[(i) >> 3] >> ((i)&7)) & ((uint8_t)1)) #define SET_BIT2(p, i, v) ((p)[(i) >> 2] &= BIT2_MAP[(i)&3][v]) From e702730790b5f6381cbfe8b0fd5e3fc67c8168eb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Aug 2022 11:36:11 +0800 Subject: [PATCH 51/92] fix: coredump --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 9fcfd9c8bd..fd39d559ea 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1198,7 +1198,6 @@ _exit: int32_t tBlockDataInitEx(SBlockData *pBlockData, SBlockData *pBlockDataFrom) { int32_t code = 0; - ASSERT(0); ASSERT(pBlockDataFrom->suid || pBlockDataFrom->uid); pBlockData->suid = pBlockDataFrom->suid; @@ -1437,7 +1436,6 @@ int32_t tBlockDataCopy(SBlockData *pSrc, SBlockData *pDest) { ASSERT(pDest->suid == pSrc->suid); ASSERT(pDest->uid == pSrc->uid); - ASSERT(pSrc->nRow == pDest->nRow); ASSERT(taosArrayGetSize(pSrc->aIdx) == taosArrayGetSize(pDest->aIdx)); pDest->nRow = pSrc->nRow; From a4e4f4cc4a7c515a9f760d568d379ce3e605eac4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Aug 2022 14:37:43 +0800 Subject: [PATCH 52/92] make it compile --- source/client/CMakeLists.txt | 4 ++-- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 16 +++++----------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index 95e86e7629..f52edbe71f 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -46,6 +46,6 @@ target_link_libraries( PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom ) -#if(${BUILD_TEST}) +if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) -#endif(${BUILD_TEST}) \ No newline at end of file +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index a490393b4c..f03f7e2a9b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -41,9 +41,9 @@ struct STsdbSnapReader { SArray* aDelData; // SArray }; -#if 0 static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; +#if 0 STsdb* pTsdb = pReader->pTsdb; while (true) { @@ -166,6 +166,7 @@ _exit: _err: tsdbError("vgId:%d, vnode snapshot tsdb read data for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, tstrerror(code)); +#endif return code; } @@ -180,7 +181,7 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { } // open - code = tsdbDelFReaderOpen(&pReader->pDelFReader, pDelFile, pTsdb, NULL); + code = tsdbDelFReaderOpen(&pReader->pDelFReader, pDelFile, pTsdb); if (code) goto _err; // read index @@ -253,11 +254,9 @@ _err: tstrerror(code)); return code; } -#endif int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, STsdbSnapReader** ppReader) { - int32_t code = 0; -#if 0 + int32_t code = 0; STsdbSnapReader* pReader = NULL; // alloc @@ -320,13 +319,11 @@ _err: tsdbError("vgId:%d, vnode snapshot tsdb reader open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, tstrerror(code)); *ppReader = NULL; -#endif return code; } int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { - int32_t code = 0; -#if 0 + int32_t code = 0; STsdbSnapReader* pReader = *ppReader; if (pReader->pDataFReader) { @@ -349,13 +346,11 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { taosMemoryFree(pReader); *ppReader = NULL; -#endif return code; } int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; -#if 0 *ppData = NULL; @@ -394,7 +389,6 @@ _exit: _err: tsdbError("vgId:%d, vnode snapshot tsdb read for %s failed since %s", TD_VID(pReader->pTsdb->pVnode), pReader->pTsdb->path, tstrerror(code)); -#endif return code; } From 117bc90dad8fb295513e82891a6c7d1728993975 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Aug 2022 15:29:42 +0800 Subject: [PATCH 53/92] more snapshot --- source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 103 ++++++++++----------- 2 files changed, 50 insertions(+), 54 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 0c7a08a2b5..4d3a074963 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -368,6 +368,7 @@ struct SSma { void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data); enum { + SNAP_DATA_CFG = 0, SNAP_DATA_META = 1, SNAP_DATA_TSDB = 2, SNAP_DATA_DEL = 3, diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index f03f7e2a9b..6ba546ecf1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -354,6 +354,7 @@ int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) { *ppData = NULL; +#if 0 // read data file if (!pReader->dataDone) { code = tsdbSnapReadData(pReader, ppData); @@ -367,6 +368,7 @@ int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) { } } } +#endif // read del file if (!pReader->delDone) { @@ -956,6 +958,7 @@ _err: tstrerror(code)); return code; } +#endif static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; @@ -966,7 +969,7 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 // reader if (pDelFile) { - code = tsdbDelFReaderOpen(&pWriter->pDelFReader, pDelFile, pTsdb, NULL); + code = tsdbDelFReaderOpen(&pWriter->pDelFReader, pDelFile, pTsdb); if (code) goto _err; code = tsdbReadDelIdx(pWriter->pDelFReader, pWriter->aDelIdxR); @@ -983,51 +986,15 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 TABLEID id = *(TABLEID*)(pData + sizeof(SSnapDataHdr)); while (true) { - SDelIdx* pDelIdx = NULL; - int64_t n = sizeof(SSnapDataHdr) + sizeof(TABLEID); - SDelData delData; - SDelIdx delIdx; - int8_t toBreak = 0; + if (pWriter->iDelIdx >= taosArrayGetSize(pWriter->aDelIdxR)) break; + if (tTABLEIDCmprFn(taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx), &id) >= 0) break; - if (pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR)) { - pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx); - } + SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx); - if (pDelIdx) { - int32_t c = tTABLEIDCmprFn(&id, pDelIdx); - if (c < 0) { - goto _new_del; - } else { - code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData); - if (code) goto _err; + code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData); + if (code) goto _err; - pWriter->iDelIdx++; - if (c == 0) { - toBreak = 1; - delIdx = (SDelIdx){.suid = id.suid, .uid = id.uid}; - goto _merge_del; - } else { - delIdx = (SDelIdx){.suid = pDelIdx->suid, .uid = pDelIdx->uid}; - goto _write_del; - } - } - } - - _new_del: - toBreak = 1; - delIdx = (SDelIdx){.suid = id.suid, .uid = id.uid}; - taosArrayClear(pWriter->aDelData); - - _merge_del: - while (n < nData) { - n += tGetDelData(pData + n, &delData); - if (taosArrayPush(pWriter->aDelData, &delData) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - } - - _write_del: + SDelIdx delIdx = *pDelIdx; code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx); if (code) goto _err; @@ -1036,7 +1003,40 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 goto _err; } - if (toBreak) break; + pWriter->iDelIdx++; + } + + if (pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR) && + tTABLEIDCmprFn(taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx), &id) == 0) { + SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx); + + code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData); + if (code) goto _err; + + pWriter->iDelIdx++; + } else { + taosArrayClear(pWriter->aDelData); + } + + int64_t n = sizeof(SSnapDataHdr) + sizeof(TABLEID); + while (n < nData) { + SDelData delData; + + n += tGetDelData(pData + n, &delData); + + if (taosArrayPush(pWriter->aDelData, &delData) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + } + + SDelIdx delIdx = {.suid = id.suid, .uid = id.uid}; + code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx); + if (code) goto _err; + + if (taosArrayPush(pWriter->aDelIdxW, &delIdx) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; } _exit: @@ -1047,9 +1047,7 @@ _err: tstrerror(code)); return code; } -#endif -#if 0 static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) { int32_t code = 0; STsdb* pTsdb = pWriter->pTsdb; @@ -1062,7 +1060,7 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) { code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData); if (code) goto _err; - SDelIdx delIdx = (SDelIdx){.suid = pDelIdx->suid, .uid = pDelIdx->uid}; + SDelIdx delIdx = *pDelIdx; code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx); if (code) goto _err; @@ -1095,11 +1093,9 @@ _err: tstrerror(code)); return code; } -#endif int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter) { - int32_t code = 0; -#if 0 + int32_t code = 0; STsdbSnapWriter* pWriter = NULL; // alloc @@ -1168,7 +1164,6 @@ _err: tsdbError("vgId:%d, tsdb snapshot writer open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, tstrerror(code)); *ppWriter = NULL; -#endif return code; } @@ -1210,10 +1205,10 @@ _err: } int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { - int32_t code = 0; -#if 0 + int32_t code = 0; SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; +#if 0 // ts data if (pHdr->type == SNAP_DATA_TSDB) { code = tsdbSnapWriteData(pWriter, pData, nData); @@ -1226,6 +1221,7 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) if (code) goto _err; } } +#endif // del data if (pHdr->type == SNAP_DATA_DEL) { @@ -1241,6 +1237,5 @@ _exit: _err: tsdbError("vgId:%d, tsdb snapshot write for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path, tstrerror(code)); -#endif return code; } From 082d2093b8e5dc11e6e6abecf0d980888fde2be0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Aug 2022 16:00:24 +0800 Subject: [PATCH 54/92] open more --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 6ba546ecf1..81c5190a32 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -354,7 +354,6 @@ int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) { *ppData = NULL; -#if 0 // read data file if (!pReader->dataDone) { code = tsdbSnapReadData(pReader, ppData); @@ -368,7 +367,6 @@ int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) { } } } -#endif // read del file if (!pReader->delDone) { From 4de48e66de8a84812e6d769436c6f03e19d574a2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Aug 2022 17:04:24 +0800 Subject: [PATCH 55/92] adjust more --- source/dnode/vnode/src/inc/tsdb.h | 1 + .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 1 + source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 37 +++++++++++++------ source/dnode/vnode/src/tsdb/tsdbUtil.c | 2 + 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 6263703c01..f41592fe46 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -502,6 +502,7 @@ struct SDelIdx { struct SDiskDataHdr { uint32_t delimiter; + uint32_t fmtVer; int64_t suid; int64_t uid; int32_t szUid; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 324db5b111..55ef576916 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1426,6 +1426,7 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock // ================= DATA ==================== SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, + .fmtVer = 0, .suid = pBlockData->suid, .uid = pBlockData->uid, .nRow = pBlockData->nRow, diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 81c5190a32..29a64cf0b5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -27,8 +27,10 @@ struct STsdbSnapReader { int32_t fid; SDataFReader* pDataFReader; SArray* aBlockIdx; // SArray + SArray* aBlockL; // SArray int32_t iBlockIdx; SBlockIdx* pBlockIdx; + int32_t iBlockL; SMapData mBlock; // SMapData int32_t iBlock; SBlockData oBlockData; @@ -43,32 +45,37 @@ struct STsdbSnapReader { static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; -#if 0 STsdb* pTsdb = pReader->pTsdb; while (true) { if (pReader->pDataFReader == NULL) { - SDFileSet* pSet = - taosArraySearch(pReader->fs.aDFileSet, &(SDFileSet){.fid = pReader->fid}, tDFileSetCmprFn, TD_GT); - + // next + SDFileSet dFileSet = {.fid = pReader->fid}; + SDFileSet* pSet = taosArraySearch(pReader->fs.aDFileSet, &dFileSet, tDFileSetCmprFn, TD_GT); if (pSet == NULL) goto _exit; - pReader->fid = pSet->fid; + + // load code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, pSet); if (code) goto _err; - // SBlockIdx code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx); if (code) goto _err; + code = tsdbReadBlockL(pReader->pDataFReader, pReader->aBlockL); + if (code) goto _err; + + // init pReader->iBlockIdx = 0; pReader->pBlockIdx = NULL; + pReader->iBlockL = 0; tsdbInfo("vgId:%d, vnode snapshot tsdb open data file to read for %s, fid:%d", TD_VID(pTsdb->pVnode), pTsdb->path, pReader->fid); } while (true) { +#if 0 if (pReader->pBlockIdx == NULL) { if (pReader->iBlockIdx >= taosArrayGetSize(pReader->aBlockIdx)) { tsdbDataFReaderClose(&pReader->pDataFReader); @@ -149,14 +156,15 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { tPutBlockData((uint8_t*)(&pId[1]), &pReader->nBlockData); - tsdbInfo("vgId:%d, vnode snapshot read data for %s, fid:%d suid:%" PRId64 " uid:%" PRId64 - " iBlock:%d minVersion:%d maxVersion:%d nRow:%d out of %d size:%d", - TD_VID(pTsdb->pVnode), pTsdb->path, pReader->fid, pReader->pBlockIdx->suid, pReader->pBlockIdx->uid, - pReader->iBlock - 1, pBlock->minVersion, pBlock->maxVersion, pReader->nBlockData.nRow, pBlock->nRow, - size); + // tsdbInfo("vgId:%d, vnode snapshot read data for %s, fid:%d suid:%" PRId64 " uid:%" PRId64 + // " iBlock:%d minVersion:%d maxVersion:%d nRow:%d out of %d size:%d", + // TD_VID(pTsdb->pVnode), pTsdb->path, pReader->fid, pReader->pBlockIdx->suid, pReader->pBlockIdx->uid, + // pReader->iBlock - 1, pBlock->minVersion, pBlock->maxVersion, pReader->nBlockData.nRow, pBlock->nRow, + // size); goto _exit; } +#endif } } @@ -166,7 +174,6 @@ _exit: _err: tsdbError("vgId:%d, vnode snapshot tsdb read data for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, tstrerror(code)); -#endif return code; } @@ -294,6 +301,11 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } + pReader->aBlockL = taosArrayInit(0, sizeof(SBlockL)); + if (pReader->aBlockL == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } pReader->mBlock = tMapDataInit(); code = tBlockDataCreate(&pReader->oBlockData); if (code) goto _err; @@ -329,6 +341,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { if (pReader->pDataFReader) { tsdbDataFReaderClose(&pReader->pDataFReader); } + taosArrayDestroy(pReader->aBlockL); taosArrayDestroy(pReader->aBlockIdx); tMapDataClear(&pReader->mBlock); tBlockDataDestroy(&pReader->oBlockData, 1); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index fd39d559ea..f324124a9c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1549,6 +1549,7 @@ int32_t tPutDiskDataHdr(uint8_t *p, void *ph) { SDiskDataHdr *pHdr = (SDiskDataHdr *)ph; n += tPutU32(p ? p + n : p, pHdr->delimiter); + n += tPutU32v(p ? p + n : p, pHdr->fmtVer); n += tPutI64(p ? p + n : p, pHdr->suid); n += tPutI64(p ? p + n : p, pHdr->uid); n += tPutI32v(p ? p + n : p, pHdr->szUid); @@ -1566,6 +1567,7 @@ int32_t tGetDiskDataHdr(uint8_t *p, void *ph) { SDiskDataHdr *pHdr = (SDiskDataHdr *)ph; n += tGetU32(p + n, &pHdr->delimiter); + n += tGetU32v(p + n, &pHdr->fmtVer); n += tGetI64(p + n, &pHdr->suid); n += tGetI64(p + n, &pHdr->uid); n += tGetI32v(p + n, &pHdr->szUid); From 8eb86676d091badf9946de6cbc285bd53446d0e8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 18 Aug 2022 17:09:44 +0800 Subject: [PATCH 56/92] fix(query): support query last file. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 173 ++++++++++++++++++++++--- 1 file changed, 156 insertions(+), 17 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 048c64032c..4af8d6e28f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1279,7 +1279,6 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBloc bool overlapWithlastBlock = false; if (hasDataInLastBlock(pLastBlockReader)) { SBlockL *pBlockL = taosArrayGet(pLastBlockReader->pBlockL, pLastBlockReader->currentBlockIndex); -// int64_t ts = getCurrentKeyInLastBlock(pLastBlockReader); overlapWithlastBlock = !(pBlock->maxKey.ts < pBlockL->minKey || pBlock->minKey.ts > pBlockL->maxKey); } @@ -1358,8 +1357,71 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* return pReader->pMemSchema; } +// todo handle desc +static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, + SIterInfo* pIter, int64_t key, SLastBlockReader* pLastBlockReader) { + SRowMerger merge = {0}; + STSRow* pTSRow = NULL; + SBlockData* pBlockData = &pReader->status.fileBlockData; + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + + int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + + TSDBKEY k = TSDBROW_KEY(pRow); + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + + SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; + + int64_t minKey = INT64_MAX; + if (minKey > tsLast) { + minKey = tsLast; + } + + if (minKey > k.ts) { + minKey = k.ts; + } + + if (minKey > key && pBlockData->nRow > 0) { + minKey = key; + } + + // file block ---> last block -----> imem -----> mem + bool init = false; + if (minKey == key) { + init = true; + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + } + + if (minKey == tsLast) { + if (!init) { + init = true; + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, pLastBlockReader->rowIndex); + tRowMergerInit(&merge, &fRow1, pReader->pSchema); + } + + doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); + } + + if (minKey == k.ts) { + if (!init) { + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + tRowMergerInit(&merge, pRow, pSchema); + } + + doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + } + + tRowMergerGetRow(&merge, &pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + + taosMemoryFree(pTSRow); + tRowMergerClear(&merge); + return TSDB_CODE_SUCCESS; +} + static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, - SIterInfo* pIter, int64_t key) { + SIterInfo* pIter, int64_t key, SLastBlockReader* pLastBlockReader) { SRowMerger merge = {0}; STSRow* pTSRow = NULL; SBlockData* pBlockData = &pReader->status.fileBlockData; @@ -1431,6 +1493,85 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return TSDB_CODE_SUCCESS; } +static int32_t doMergeThreeLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { + SRowMerger merge = {0}; + STSRow* pTSRow = NULL; + + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + SArray* pDelList = pBlockScanInfo->delSkyline; + + TSDBROW* pRow = getValidRow(&pBlockScanInfo->iter, pDelList, pReader); + TSDBROW* piRow = getValidRow(&pBlockScanInfo->iiter, pDelList, pReader); + ASSERT(pRow != NULL && piRow != NULL); + + SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; + int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + + int64_t key = pBlockData->aTSKEY[pDumpInfo->rowIndex]; + + TSDBKEY k = TSDBROW_KEY(pRow); + TSDBKEY ik = TSDBROW_KEY(piRow); + + int64_t minKey = INT64_MAX; + if (minKey > k.ts) { + minKey = k.ts; + } + + if (minKey > ik.ts) { + minKey = ik.ts; + } + + if (minKey > key) { + minKey = key; + } + + if (minKey > tsLast) { + minKey = tsLast; + } + + // file block ---> last block -----> imem -----> mem + bool init = false; + if (minKey == key) { + init = true; + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + } + + if (minKey == tsLast) { + if (!init) { + init = true; + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, pLastBlockReader->rowIndex); + tRowMergerInit(&merge, &fRow1, pReader->pSchema); + } + doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); + } + + if (minKey == ik.ts) { + if (!init) { + init = true; + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); + tRowMergerInit(&merge, piRow, pSchema); + } + doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); + } + + if (minKey == k.ts) { + if (!init) { + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + tRowMergerInit(&merge, pRow, pSchema); + } + doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + } + + tRowMergerGetRow(&merge, &pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + + taosMemoryFree(pTSRow); + tRowMergerClear(&merge); + return TSDB_CODE_SUCCESS; +} + static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) { SRowMerger merge = {0}; STSRow* pTSRow = NULL; @@ -1605,8 +1746,8 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } -static void initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid, STimeWindow* pWin, SVersionRange* pVerRange, - int16_t startPos) { +static void initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid, STimeWindow* pWin, + SVersionRange* pVerRange, int16_t startPos) { pLastBlockReader->uid = uid; pLastBlockReader->window = *pWin; pLastBlockReader->verRange = *pVerRange; @@ -1636,10 +1777,12 @@ static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { // no data any more if (pBlockData->aTSKEY[i] > pLastBlockReader->window.ekey) { + pLastBlockReader->rowIndex = pBlockData->nRow; return false; } if (pBlockData->aVersion[i] > pLastBlockReader->verRange.maxVer) { + pLastBlockReader->rowIndex = pBlockData->nRow; return false; } @@ -1676,25 +1819,21 @@ static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader *pLastBlockReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - int64_t key = INT64_MIN; - if (pBlockData->nRow > 0) { - key = pBlockData->aTSKEY[pDumpInfo->rowIndex]; - } - + int64_t key = (pBlockData->nRow > 0)? pBlockData->aTSKEY[pDumpInfo->rowIndex]:INT64_MIN; TSDBROW* pRow = getValidRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); TSDBROW* piRow = getValidRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); if (pBlockScanInfo->iter.hasVal && pBlockScanInfo->iiter.hasVal) { - return doMergeThreeLevelRows(pReader, pBlockScanInfo, pBlockData); + return doMergeThreeLevelRowsRv(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); } else { - // imem + file + // imem + file + last block if (pBlockScanInfo->iiter.hasVal) { - return doMergeBufAndFileRows(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, key); + return doMergeBufAndFileRows_Rv(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, key, pLastBlockReader); } // mem + file if (pBlockScanInfo->iter.hasVal) { - return doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, key); + return doMergeBufAndFileRows_Rv(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, key, pLastBlockReader); } if (pBlockData->nRow > 0) { @@ -1704,7 +1843,6 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI int64_t ts = getCurrentKeyInLastBlock(pLastBlockReader); if (ts < key) { // save rows in last block SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; - int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); STSRow* pTSRow = NULL; SRowMerger merge = {0}; @@ -1712,7 +1850,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, pLastBlockReader->rowIndex); tRowMergerInit(&merge, &fRow1, pReader->pSchema); - doMergeRowsInLastBlock(pLastBlockReader, tsLastBlock, &merge); + doMergeRowsInLastBlock(pLastBlockReader, ts, &merge); tRowMergerGetRow(&merge, &pTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); @@ -1753,6 +1891,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI } } } else { // only last block exists + // only last block exits SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); @@ -1820,10 +1959,10 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } buildComposedDataBlockImpl(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); - SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); // currently loaded file data block is consumed - if (pBlockData->nRow > 0 && (pDumpInfo->rowIndex >= pBlock->nRow || pDumpInfo->rowIndex < 0)) { + if (pBlockData->nRow > 0 && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { + SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); setBlockAllDumped(pDumpInfo, pBlock, pReader->order); break; } From 871b4895f92dd74af5559dbfc838e2db09cd3963 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Aug 2022 18:26:10 +0800 Subject: [PATCH 57/92] fix bit map bug --- include/common/tdataformat.h | 4 ++-- source/dnode/vnode/src/tsdb/tsdbUtil.c | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index e94edfcb1e..1cc226f9d7 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -50,9 +50,9 @@ const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, #define BIT1_SIZE(n) ((((n)-1) >> 3) + 1) #define BIT2_SIZE(n) ((((n)-1) >> 2) + 1) -#define SET_BIT1(p, i, v) ((p)[(i) >> 3] &= BIT1_MAP[(i)&7][v]) +#define SET_BIT1(p, i, v) ((p)[(i) >> 3] |= BIT1_MAP[(i)&7][v]) #define GET_BIT1(p, i) (((p)[(i) >> 3] >> ((i)&7)) & ((uint8_t)1)) -#define SET_BIT2(p, i, v) ((p)[(i) >> 2] &= BIT2_MAP[(i)&3][v]) +#define SET_BIT2(p, i, v) ((p)[(i) >> 2] |= BIT2_MAP[(i)&3][v]) #define GET_BIT2(p, i) (((p)[(i) >> 2] >> BIT2_MAP[(i)&3][3]) & ((uint8_t)3)) // STSchema diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index f324124a9c..ad759f1ef8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -928,6 +928,9 @@ int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) { size = BIT2_SIZE(pColData->nVal + 1); code = tRealloc(&pColData->pBitMap, size); if (code) goto _exit; + if ((pColData->nVal & 3) == 0) { + pColData->pBitMap[pColData->nVal >> 2] = 0; + } // put value if (pColVal->isNone) { From e75c4000454e70f1e68c5d1fc73f3762d673150c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 18 Aug 2022 18:48:50 +0800 Subject: [PATCH 58/92] fix(query): fix bug in query last block. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 155 +++++++++++++++---------- 1 file changed, 93 insertions(+), 62 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index bc20240b4c..f4671a0f34 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -89,7 +89,7 @@ typedef struct SLastBlockReader { STimeWindow window; SVersionRange verRange; uint64_t uid; - int32_t rowIndex; + int16_t* rowIndex; // row index ptr, usually from the STableBlockScanInfo->indexInBlockL } SLastBlockReader; typedef struct SFilesetIter { @@ -181,7 +181,8 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* re int8_t* pLevel); static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level); static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader); -static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); +static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); +static int32_t doBuildDataBlock(STsdbReader* pReader); static int32_t setColumnIdSlotList(STsdbReader* pReader, SSDataBlock* pBlock) { SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; @@ -449,7 +450,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->pTsdb = getTsdbByRetentions(pVnode, pCond->twindows.skey, pVnode->config.tsdbCfg.retentions, idstr, &level); pReader->suid = pCond->suid; pReader->order = pCond->order; - pReader->capacity = capacity; + pReader->capacity = 1; pReader->idStr = (idstr != NULL) ? strdup(idstr) : NULL; pReader->verRange = getQueryVerRange(pVnode, pCond, level); pReader->type = pCond->type; @@ -624,24 +625,29 @@ _end: return code; } -static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* pLastBlockIndex, - SBlockNumber * pBlockNum, SArray* pQualifiedLastBlock) { - int32_t numOfQTable= 0; - size_t numOfTables = taosArrayGetSize(pIndexList); - - int64_t st = taosGetTimestampUs(); - size_t size = 0; - +static void cleanupTableScanInfo(SHashObj* pTableMap) { STableBlockScanInfo* px = NULL; while (1) { - px = taosHashIterate(pReader->status.pTableMap, px); + px = taosHashIterate(pTableMap, px); if (px == NULL) { break; } + // reset the index in last block when handing a new file + px->indexInBlockL = -1; tMapDataClear(&px->mapData); taosArrayClear(px->pBlockList); } +} + +static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* pLastBlockIndex, + SBlockNumber * pBlockNum, SArray* pQualifiedLastBlock) { + int32_t numOfQTable = 0; + size_t sizeInDisk = 0; + size_t numOfTables = taosArrayGetSize(pIndexList); + + int64_t st = taosGetTimestampUs(); + cleanupTableScanInfo(pReader->status.pTableMap); for (int32_t i = 0; i < numOfTables; ++i) { SBlockIdx* pBlockIdx = taosArrayGet(pIndexList, i); @@ -651,7 +657,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* tMapDataReset(&pScanInfo->mapData); tsdbReadBlock(pReader->pFileReader, pBlockIdx, &pScanInfo->mapData); - size += pScanInfo->mapData.nData; + sizeInDisk += pScanInfo->mapData.nData; for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) { SBlock block = {0}; tMapDataGetItemByIdx(&pScanInfo->mapData, j, &block, tGetBlock); @@ -707,7 +713,8 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* double el = (taosGetTimestampUs() - st) / 1000.0; tsdbDebug("load block of %d tables completed, blocks:%d in %d tables, lastBlock:%d, size:%.2f Kb, elapsed time:%.2f ms %s", - numOfTables, total, numOfQTable, pBlockNum->numOfLastBlocks, size / 1000.0, el, pReader->idStr); + numOfTables, total, numOfQTable, pBlockNum->numOfLastBlocks, sizeInDisk + / 1000.0, el, pReader->idStr); pReader->cost.numOfBlocks += total; pReader->cost.headFileLoadTime += el; @@ -1396,7 +1403,7 @@ static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInf if (minKey == tsLast) { if (!init) { init = true; - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, pLastBlockReader->rowIndex); + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); tRowMergerInit(&merge, &fRow1, pReader->pSchema); } @@ -1541,7 +1548,7 @@ static int32_t doMergeThreeLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo if (minKey == tsLast) { if (!init) { init = true; - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, pLastBlockReader->rowIndex); + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); tRowMergerInit(&merge, &fRow1, pReader->pSchema); } doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); @@ -1747,7 +1754,7 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } static void initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid, STimeWindow* pWin, - SVersionRange* pVerRange, int16_t startPos) { + SVersionRange* pVerRange, int16_t* startPos) { pLastBlockReader->uid = uid; pLastBlockReader->window = *pWin; pLastBlockReader->verRange = *pVerRange; @@ -1755,14 +1762,14 @@ static void initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid } static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { - if (pLastBlockReader->rowIndex >= pLastBlockReader->lastBlockData.nRow) { + if (*(pLastBlockReader->rowIndex) >= pLastBlockReader->lastBlockData.nRow) { return false; } - pLastBlockReader->rowIndex += 1; + *(pLastBlockReader->rowIndex) += 1; SBlockData* pBlockData = &pLastBlockReader->lastBlockData; - for(int32_t i = pLastBlockReader->rowIndex; i < pBlockData->nRow; ++i) { + for(int32_t i = *(pLastBlockReader->rowIndex); i < pBlockData->nRow; ++i) { if (pBlockData->aUid[i] != pLastBlockReader->uid) { continue; } @@ -1777,23 +1784,25 @@ static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { // no data any more if (pBlockData->aTSKEY[i] > pLastBlockReader->window.ekey) { - pLastBlockReader->rowIndex = pBlockData->nRow; + *(pLastBlockReader->rowIndex) = pBlockData->nRow; return false; } if (pBlockData->aVersion[i] > pLastBlockReader->verRange.maxVer) { - pLastBlockReader->rowIndex = pBlockData->nRow; + *(pLastBlockReader->rowIndex) = pBlockData->nRow; return false; } - pLastBlockReader->rowIndex = i; + *(pLastBlockReader->rowIndex) = i; return true; } - pLastBlockReader->rowIndex = pBlockData->nRow; + // set all data is consumed in last block + *(pLastBlockReader->rowIndex) = pBlockData->nRow; return false; } +#if 0 static int32_t saveCurrentState(SLastBlockReader* pLastBlockReader) { return pLastBlockReader->rowIndex; } @@ -1801,15 +1810,16 @@ static int32_t saveCurrentState(SLastBlockReader* pLastBlockReader) { static void restoreState(SLastBlockReader* pLastBlockReader, int32_t state) { pLastBlockReader->rowIndex = state; } +#endif static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { SBlockData* pBlockData = &pLastBlockReader->lastBlockData; - return pBlockData->aTSKEY[pLastBlockReader->rowIndex]; + return pBlockData->aTSKEY[*pLastBlockReader->rowIndex]; } // todo handle desc order static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { - if (pLastBlockReader->rowIndex >= pLastBlockReader->lastBlockData.nRow) { + if (*pLastBlockReader->rowIndex >= pLastBlockReader->lastBlockData.nRow) { return false; } @@ -1847,7 +1857,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI STSRow* pTSRow = NULL; SRowMerger merge = {0}; - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, pLastBlockReader->rowIndex); + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); tRowMergerInit(&merge, &fRow1, pReader->pSchema); doMergeRowsInLastBlock(pLastBlockReader, ts, &merge); @@ -1898,7 +1908,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI STSRow* pTSRow = NULL; SRowMerger merge = {0}; - TSDBROW fRow = tsdbRowFromBlockData(pLastBlockData, pLastBlockReader->rowIndex); + TSDBROW fRow = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); tRowMergerInit(&merge, &fRow, pReader->pSchema); doMergeRowsInLastBlock(pLastBlockReader, tsLastBlock, &merge); @@ -1925,12 +1935,9 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { pBlockScanInfo = pReader->status.pTableIter; } - SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; - - int16_t startIndex = pBlockInfo != NULL? pBlockScanInfo->indexInBlockL:-1; - initLastBlockReader(pLastBlockReader, pBlockScanInfo->uid, &pReader->window, &pReader->verRange, startIndex); - - bool has = nextRowInLastBlock(pLastBlockReader); // todo handle if no data in last block + SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + initLastBlockReader(pLastBlockReader, pBlockScanInfo->uid, &pReader->window, &pReader->verRange, &pBlockScanInfo->indexInBlockL); +// bool has = nextRowInLastBlock(pLastBlockReader); // todo handle if no data in last block SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBlockData* pBlockData = &pReader->status.fileBlockData; @@ -1961,7 +1968,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { buildComposedDataBlockImpl(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); // currently loaded file data block is consumed - if (pBlockData->nRow > 0 && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { + if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); setBlockAllDumped(pDumpInfo, pBlock, pReader->order); break; @@ -2252,7 +2259,7 @@ static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, uint64 } static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { - SReaderStatus* pStatus = &pReader->status; + SReaderStatus* pStatus = &pReader->status; SLastBlockReader* pLastBlockReader = pStatus->fileIter.pLastBlockReader; while(1) { @@ -2261,17 +2268,42 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { if (pStatus->pTableIter == NULL) { return TSDB_CODE_SUCCESS; } - } else { // let's try next table - pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); - if (pStatus->pTableIter == NULL) { - return TSDB_CODE_SUCCESS; + } + + // load the last data block of current table + // todo opt perf by avoiding load last block repeatly + STableBlockScanInfo* pScanInfo = pStatus->pTableIter; + int32_t code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo->uid, pReader); + if (code != TSDB_CODE_SUCCESS) { // todo handle error + return code; + } + + initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pReader->window, &pReader->verRange, &pScanInfo->indexInBlockL); + if (pScanInfo->indexInBlockL == -1) { + bool hasData = nextRowInLastBlock(pLastBlockReader); + if (!hasData) { // current table does not have rows in last block, try next table + pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); + if (pStatus->pTableIter == NULL) { + return TSDB_CODE_SUCCESS; + } + continue; } } - // find the last block that contain the specified block uid - return doLoadRelatedLastBlock(pLastBlockReader, pStatus->pTableIter->uid, pReader); + code = doBuildDataBlock(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - //todo check for all empty table + if (pReader->pResBlock->info.rows > 0) { + return TSDB_CODE_SUCCESS; + } + + // current table is exhausted, let's try next table + pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); + if (pStatus->pTableIter == NULL) { + return TSDB_CODE_SUCCESS; + } } } @@ -2281,8 +2313,8 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; - SBlock* pBlock = NULL; TSDBKEY key = {0}; + SBlock* pBlock = NULL; STableBlockScanInfo* pScanInfo = NULL; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; @@ -2292,29 +2324,28 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { pBlock = getCurrentBlock(pBlockIter); key = getCurrentKeyInBuf(pBlockIter, pReader); + // load the last data block of current table code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo->uid, pReader); if (code != TSDB_CODE_SUCCESS) { // todo handle error } - initLastBlockReader(pLastBlockReader, pBlockInfo->uid, &pReader->window, &pReader->verRange, pScanInfo->indexInBlockL); - bool hasData = nextRowInLastBlock(pLastBlockReader); - } else { - ASSERT(pBlockIter->numOfBlocks == 0); + initLastBlockReader(pLastBlockReader, pBlockInfo->uid, &pReader->window, &pReader->verRange, &pScanInfo->indexInBlockL); } - if (pBlockInfo == NULL || fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, key, pLastBlockReader)) { - if (pBlockInfo != NULL) { - tBlockDataReset(&pStatus->fileBlockData); - code = tBlockDataInit(&pStatus->fileBlockData, pReader->suid, pScanInfo->uid, pReader->pSchema); - if (code != TSDB_CODE_SUCCESS) { - //todo - } + if (pBlockInfo == NULL) { // build data block from last data file + ASSERT(pBlockIter->numOfBlocks == 0); + code = buildComposedDataBlock(pReader); + } else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, key, pLastBlockReader)) { + tBlockDataReset(&pStatus->fileBlockData); + code = tBlockDataInit(&pStatus->fileBlockData, pReader->suid, pScanInfo->uid, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + // todo + } - code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData); + if (code != TSDB_CODE_SUCCESS) { + return code; } // build composed data block @@ -2425,7 +2456,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { return code; } - // all data blocks are check in last file, now let's try the next file + // all data blocks are checked in this last block file, now let's try the next file if (pReader->status.pTableIter == NULL) { code = initForFirstBlockInFile(pReader, pBlockIter); @@ -2434,7 +2465,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { return code; } - // this file does not have blocks, let's start check the last block file + // this file does not have data files, let's start check the last block file if exists if (pBlockIter->numOfBlocks == 0) { goto _begin; } @@ -2855,7 +2886,7 @@ int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, int64_t ts, S while(nextRowInLastBlock(pLastBlockReader)) { int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 == ts) { - TSDBROW fRow1 = tsdbRowFromBlockData(&pLastBlockReader->lastBlockData, pLastBlockReader->rowIndex); + TSDBROW fRow1 = tsdbRowFromBlockData(&pLastBlockReader->lastBlockData, *pLastBlockReader->rowIndex); tRowMerge(pMerger, &fRow1); } else { break; From 873db889026d58366f8b4868d63a93ea339dd1b4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Aug 2022 19:37:05 +0800 Subject: [PATCH 59/92] fix --- include/common/tdataformat.h | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 1cc226f9d7..fc25e42144 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -38,19 +38,15 @@ typedef struct STagVal STagVal; typedef struct STag STag; // bitmap -const static uint8_t BIT1_MAP[8][2] = { - {0b00000000, 0b00000001}, {0b00000001, 0b00000011}, {0b00000011, 0b00000111}, {0b00000111, 0b00001111}, - {0b00001111, 0b00011111}, {0b00011111, 0b00111111}, {0b00111111, 0b01111111}, {0b01111111, 0b11111111}, -}; - const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, - {0b00000011, 0b00000111, 0b00001011, 2}, - {0b00001111, 0b00011111, 0b00101111, 4}, - {0b00111111, 0b01111111, 0b10111111, 6}}; + {0b00000000, 0b00000100, 0b00001000, 2}, + {0b00000000, 0b00010000, 0b00100000, 4}, + {0b00000000, 0b01000000, 0b10000000, 6}}; -#define BIT1_SIZE(n) ((((n)-1) >> 3) + 1) -#define BIT2_SIZE(n) ((((n)-1) >> 2) + 1) -#define SET_BIT1(p, i, v) ((p)[(i) >> 3] |= BIT1_MAP[(i)&7][v]) +#define BIT1_SIZE(n) ((((n)-1) >> 3) + 1) +#define BIT2_SIZE(n) ((((n)-1) >> 2) + 1) +#define SET_BIT1(p, i, v) \ + if (v) (p)[(i) >> 3] |= (((uint8_t)1) << ((i)&7)) #define GET_BIT1(p, i) (((p)[(i) >> 3] >> ((i)&7)) & ((uint8_t)1)) #define SET_BIT2(p, i, v) ((p)[(i) >> 2] |= BIT2_MAP[(i)&3][v]) #define GET_BIT2(p, i) (((p)[(i) >> 2] >> BIT2_MAP[(i)&3][3]) & ((uint8_t)3)) From c7d7cb7d5f6c72519f77ef13fcdc55cd4df362e8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Aug 2022 19:40:28 +0800 Subject: [PATCH 60/92] more fix --- include/common/tdataformat.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index fc25e42144..b9ce7368a7 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -43,10 +43,9 @@ const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, {0b00000000, 0b00010000, 0b00100000, 4}, {0b00000000, 0b01000000, 0b10000000, 6}}; -#define BIT1_SIZE(n) ((((n)-1) >> 3) + 1) -#define BIT2_SIZE(n) ((((n)-1) >> 2) + 1) -#define SET_BIT1(p, i, v) \ - if (v) (p)[(i) >> 3] |= (((uint8_t)1) << ((i)&7)) +#define BIT1_SIZE(n) ((((n)-1) >> 3) + 1) +#define BIT2_SIZE(n) ((((n)-1) >> 2) + 1) +#define SET_BIT1(p, i, v) ((p)[(i) >> 3] |= (((uint8_t)v) << ((i)&7))) #define GET_BIT1(p, i) (((p)[(i) >> 3] >> ((i)&7)) & ((uint8_t)1)) #define SET_BIT2(p, i, v) ((p)[(i) >> 2] |= BIT2_MAP[(i)&3][v]) #define GET_BIT2(p, i) (((p)[(i) >> 2] >> BIT2_MAP[(i)&3][3]) & ((uint8_t)3)) From eaef3dffafb9d991773acba0c0ac375fd19d4d92 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 18 Aug 2022 22:42:16 +0800 Subject: [PATCH 61/92] fix(query): fix error in query last block. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 163 +++++++++++++++++-------- 1 file changed, 112 insertions(+), 51 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index f4671a0f34..5284b6936a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -88,6 +88,7 @@ typedef struct SLastBlockReader { SBlockData lastBlockData; STimeWindow window; SVersionRange verRange; + int32_t order; uint64_t uid; int16_t* rowIndex; // row index ptr, usually from the STableBlockScanInfo->indexInBlockL } SLastBlockReader; @@ -313,11 +314,11 @@ static void limitOutputBufferSize(const SQueryTableDataCond* pCond, int32_t* cap } // init file iterator -static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, int32_t order, const char* idstr) { +static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdbReader* pReader/*int32_t order, const char* idstr*/) { size_t numOfFileset = taosArrayGetSize(aDFileSet); - pIter->index = ASCENDING_TRAVERSE(order) ? -1 : numOfFileset; - pIter->order = order; + pIter->index = ASCENDING_TRAVERSE(pReader->order) ? -1 : numOfFileset; + pIter->order = pReader->order; pIter->pFileList = aDFileSet; pIter->numOfFiles = numOfFileset; @@ -325,14 +326,18 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, int32 pIter->pLastBlockReader = taosMemoryCalloc(1, sizeof(struct SLastBlockReader)); if (pIter->pLastBlockReader == NULL) { int32_t code = TSDB_CODE_OUT_OF_MEMORY; - tsdbError("failed to prepare the last block iterator, code:%d %s", tstrerror(code), idstr); + tsdbError("failed to prepare the last block iterator, code:%d %s", tstrerror(code), pReader->idStr); return code; } - pIter->pLastBlockReader->pBlockL = taosArrayInit(4, sizeof(SBlockL)); + SLastBlockReader* pLReader = pIter->pLastBlockReader; + pLReader->pBlockL = taosArrayInit(4, sizeof(SBlockL)); + pLReader->order = pReader->order; + pLReader->window = pReader->window; + pLReader->verRange = pReader->verRange; } - tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, idstr); + tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, pReader->idStr); return TSDB_CODE_SUCCESS; } @@ -1284,7 +1289,7 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBloc // todo here we need to each key in the last files to identify if it is really overlapped with last block bool overlapWithlastBlock = false; - if (hasDataInLastBlock(pLastBlockReader)) { + if (/*hasDataInLastBlock(pLastBlockReader)*/taosArrayGetSize(pLastBlockReader->pBlockL) > 0) { SBlockL *pBlockL = taosArrayGet(pLastBlockReader->pBlockL, pLastBlockReader->currentBlockIndex); overlapWithlastBlock = !(pBlock->maxKey.ts < pBlockL->minKey || pBlock->minKey.ts > pBlockL->maxKey); } @@ -1364,7 +1369,6 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* return pReader->pMemSchema; } -// todo handle desc static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, SIterInfo* pIter, int64_t key, SLastBlockReader* pLastBlockReader) { SRowMerger merge = {0}; @@ -1392,31 +1396,57 @@ static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInf minKey = key; } - // file block ---> last block -----> imem -----> mem bool init = false; - if (minKey == key) { - init = true; - tRowMergerInit(&merge, &fRow, pReader->pSchema); - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - } - if (minKey == tsLast) { - if (!init) { + // file block ---> last block -----> imem -----> mem + if (pReader->order == TSDB_ORDER_ASC) { + if (minKey == key) { init = true; - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); - tRowMergerInit(&merge, &fRow1, pReader->pSchema); + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } - doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); - } + if (minKey == tsLast) { + if (!init) { + init = true; + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + tRowMergerInit(&merge, &fRow1, pReader->pSchema); + } + doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); + } - if (minKey == k.ts) { - if (!init) { + if (minKey == k.ts) { + if (!init) { + init = true; + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + tRowMergerInit(&merge, pRow, pSchema); + } + doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + } + } else { + if (minKey == k.ts) { + init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); tRowMergerInit(&merge, pRow, pSchema); + doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); } - doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + if (minKey == tsLast) { + if (!init) { + init = true; + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + tRowMergerInit(&merge, &fRow1, pReader->pSchema); + } + doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); + } + + if (minKey == key) { + if (!init) { + init = true; + tRowMergerInit(&merge, &fRow, pReader->pSchema); + } + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + } } tRowMergerGetRow(&merge, &pTSRow); @@ -1753,23 +1783,34 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } -static void initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid, STimeWindow* pWin, - SVersionRange* pVerRange, int16_t* startPos) { +static void initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid, int16_t* startPos) { pLastBlockReader->uid = uid; - pLastBlockReader->window = *pWin; - pLastBlockReader->verRange = *pVerRange; pLastBlockReader->rowIndex = startPos; + + if (*startPos == -1) { + if (ASCENDING_TRAVERSE(pLastBlockReader->order)) { + // do nothing + } else { + *startPos = pLastBlockReader->lastBlockData.nRow; + } + } +} + +#define ALL_ROWS_CHECKED_INDEX INT16_MIN +static void setAllRowsChecked(SLastBlockReader *pLastBlockReader) { + *pLastBlockReader->rowIndex = ALL_ROWS_CHECKED_INDEX; } static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { - if (*(pLastBlockReader->rowIndex) >= pLastBlockReader->lastBlockData.nRow) { + int32_t step = (pLastBlockReader->order == TSDB_ORDER_ASC) ? 1 : -1; + if (*pLastBlockReader->rowIndex == ALL_ROWS_CHECKED_INDEX) { return false; } - *(pLastBlockReader->rowIndex) += 1; + *(pLastBlockReader->rowIndex) += step; SBlockData* pBlockData = &pLastBlockReader->lastBlockData; - for(int32_t i = *(pLastBlockReader->rowIndex); i < pBlockData->nRow; ++i) { + for(int32_t i = *(pLastBlockReader->rowIndex); i < pBlockData->nRow && i >= 0; i += step) { if (pBlockData->aUid[i] != pLastBlockReader->uid) { continue; } @@ -1784,12 +1825,12 @@ static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { // no data any more if (pBlockData->aTSKEY[i] > pLastBlockReader->window.ekey) { - *(pLastBlockReader->rowIndex) = pBlockData->nRow; + setAllRowsChecked(pLastBlockReader); return false; } if (pBlockData->aVersion[i] > pLastBlockReader->verRange.maxVer) { - *(pLastBlockReader->rowIndex) = pBlockData->nRow; + setAllRowsChecked(pLastBlockReader); return false; } @@ -1798,7 +1839,7 @@ static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { } // set all data is consumed in last block - *(pLastBlockReader->rowIndex) = pBlockData->nRow; + setAllRowsChecked(pLastBlockReader); return false; } @@ -1817,15 +1858,14 @@ static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { return pBlockData->aTSKEY[*pLastBlockReader->rowIndex]; } -// todo handle desc order static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { - if (*pLastBlockReader->rowIndex >= pLastBlockReader->lastBlockData.nRow) { + if (*pLastBlockReader->rowIndex == ALL_ROWS_CHECKED_INDEX) { return false; } - return true; } +// todo refactor static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader *pLastBlockReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; @@ -1849,6 +1889,25 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI if (pBlockData->nRow > 0) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + // no last block + if (pLastBlockReader->lastBlockData.nRow == 0) { + if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { + return TSDB_CODE_SUCCESS; + } else { + STSRow* pTSRow = NULL; + SRowMerger merge = {0}; + + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + tRowMergerGetRow(&merge, &pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + + taosMemoryFree(pTSRow); + tRowMergerClear(&merge); + return TSDB_CODE_SUCCESS; + } + } + // row in last file block int64_t ts = getCurrentKeyInLastBlock(pLastBlockReader); if (ts < key) { // save rows in last block @@ -1901,7 +1960,6 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI } } } else { // only last block exists - // only last block exits SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); @@ -1936,7 +1994,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; - initLastBlockReader(pLastBlockReader, pBlockScanInfo->uid, &pReader->window, &pReader->verRange, &pBlockScanInfo->indexInBlockL); + initLastBlockReader(pLastBlockReader, pBlockScanInfo->uid, &pBlockScanInfo->indexInBlockL); // bool has = nextRowInLastBlock(pLastBlockReader); // todo handle if no data in last block SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; @@ -2274,12 +2332,12 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { // todo opt perf by avoiding load last block repeatly STableBlockScanInfo* pScanInfo = pStatus->pTableIter; int32_t code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo->uid, pReader); - if (code != TSDB_CODE_SUCCESS) { // todo handle error + if (code != TSDB_CODE_SUCCESS) { return code; } - initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pReader->window, &pReader->verRange, &pScanInfo->indexInBlockL); - if (pScanInfo->indexInBlockL == -1) { + initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); + if (pScanInfo->indexInBlockL == -1 || pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) { bool hasData = nextRowInLastBlock(pLastBlockReader); if (!hasData) { // current table does not have rows in last block, try next table pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); @@ -2327,10 +2385,10 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // load the last data block of current table code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo->uid, pReader); if (code != TSDB_CODE_SUCCESS) { - // todo handle error + return code; } - initLastBlockReader(pLastBlockReader, pBlockInfo->uid, &pReader->window, &pReader->verRange, &pScanInfo->indexInBlockL); + initLastBlockReader(pLastBlockReader, pBlockInfo->uid, &pScanInfo->indexInBlockL); } if (pBlockInfo == NULL) { // build data block from last data file @@ -2340,7 +2398,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { tBlockDataReset(&pStatus->fileBlockData); code = tBlockDataInit(&pStatus->fileBlockData, pReader->suid, pScanInfo->uid, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { - // todo + return code; } code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData); @@ -2456,6 +2514,10 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { return code; } + if (pReader->pResBlock->info.rows > 0) { + return TSDB_CODE_SUCCESS; + } + // all data blocks are checked in this last block file, now let's try the next file if (pReader->status.pTableIter == NULL) { code = initForFirstBlockInFile(pReader, pBlockIter); @@ -2881,7 +2943,7 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc return TSDB_CODE_SUCCESS; } -// todo support desc order +// todo check if the rows are dropped or not int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, int64_t ts, SRowMerger* pMerger) { while(nextRowInLastBlock(pLastBlockReader)) { int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); @@ -3240,7 +3302,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) { SDataBlockIter* pBlockIter = &pReader->status.blockIter; - initFilesetIterator(&pReader->status.fileIter, pReader->pReadSnap->fs.aDFileSet, pReader->order, pReader->idStr); + initFilesetIterator(&pReader->status.fileIter, pReader->pReadSnap->fs.aDFileSet, pReader); resetDataBlockIterator(&pReader->status.blockIter, pReader->order, pReader->status.pTableMap); // no data in files, let's try buffer in memory @@ -3261,8 +3323,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl goto _err; } - initFilesetIterator(&pPrevReader->status.fileIter, pPrevReader->pReadSnap->fs.aDFileSet, pPrevReader->order, - pPrevReader->idStr); + initFilesetIterator(&pPrevReader->status.fileIter, pPrevReader->pReadSnap->fs.aDFileSet, pPrevReader); resetDataBlockIterator(&pPrevReader->status.blockIter, pPrevReader->order, pReader->status.pTableMap); // no data in files, let's try buffer in memory @@ -3507,13 +3568,13 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) { tBlockDataReset(&pStatus->fileBlockData); int32_t code = tBlockDataInit(&pStatus->fileBlockData, pReader->suid, pBlockScanInfo->uid, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { - //todo + terrno = code; + return NULL; } code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { tBlockDataDestroy(&pStatus->fileBlockData, 1); - terrno = code; return NULL; } @@ -3555,7 +3616,7 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { int32_t numOfTables = taosHashGetSize(pReader->status.pTableMap); tsdbDataFReaderClose(&pReader->pFileReader); - initFilesetIterator(&pReader->status.fileIter, pReader->pReadSnap->fs.aDFileSet, pReader->order, pReader->idStr); + initFilesetIterator(&pReader->status.fileIter, pReader->pReadSnap->fs.aDFileSet, pReader); resetDataBlockIterator(&pReader->status.blockIter, pReader->order, pReader->status.pTableMap); resetDataBlockScanInfo(pReader->status.pTableMap); From 5eae03fcab5533d1a0e9b1cfe42dca9f328f27c0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 18 Aug 2022 22:57:26 +0800 Subject: [PATCH 62/92] fix(query): handle the reverse case. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 40 +++++++++++++++++++------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5284b6936a..db943645e4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1376,24 +1376,43 @@ static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInf SBlockData* pBlockData = &pReader->status.fileBlockData; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + int64_t tsLast = INT64_MIN; + if (pLastBlockReader->lastBlockData.nRow > 0) { + tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + } TSDBKEY k = TSDBROW_KEY(pRow); TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; - int64_t minKey = INT64_MAX; - if (minKey > tsLast) { - minKey = tsLast; - } + int64_t minKey = 0; + if (pReader->order == TSDB_ORDER_ASC) { + minKey = INT64_MAX; // chosen the minimum value + if (minKey > tsLast && pLastBlockReader->lastBlockData.nRow > 0) { + minKey = tsLast; + } - if (minKey > k.ts) { - minKey = k.ts; - } + if (minKey > k.ts) { + minKey = k.ts; + } - if (minKey > key && pBlockData->nRow > 0) { - minKey = key; + if (minKey > key && pBlockData->nRow > 0) { + minKey = key; + } + } else { + minKey = INT64_MIN; + if (minKey < tsLast && pLastBlockReader->lastBlockData.nRow > 0) { + minKey = tsLast; + } + + if (minKey < k.ts) { + minKey = k.ts; + } + + if (minKey < key && pBlockData->nRow > 0) { + minKey = key; + } } bool init = false; @@ -1530,6 +1549,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return TSDB_CODE_SUCCESS; } +// todo handle the desc order check static int32_t doMergeThreeLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { SRowMerger merge = {0}; STSRow* pTSRow = NULL; From fc0745825e4cb7b30e1a12f118bbf8d80c9750b8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 09:51:40 +0800 Subject: [PATCH 63/92] more snapshot --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 132 +++++++-------------- 1 file changed, 40 insertions(+), 92 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 29a64cf0b5..dbf710bb20 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -28,13 +28,15 @@ struct STsdbSnapReader { SDataFReader* pDataFReader; SArray* aBlockIdx; // SArray SArray* aBlockL; // SArray - int32_t iBlockIdx; SBlockIdx* pBlockIdx; - int32_t iBlockL; - SMapData mBlock; // SMapData - int32_t iBlock; - SBlockData oBlockData; - SBlockData nBlockData; + SBlockL* pBlockL; + + int32_t iBlockIdx; + int32_t iBlockL; + SMapData mBlock; // SMapData + int32_t iBlock; + SBlockData oBlockData; + SBlockData nBlockData; // for del file int8_t delDone; SDelFReader* pDelFReader; @@ -75,96 +77,42 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { } while (true) { -#if 0 - if (pReader->pBlockIdx == NULL) { - if (pReader->iBlockIdx >= taosArrayGetSize(pReader->aBlockIdx)) { - tsdbDataFReaderClose(&pReader->pDataFReader); - break; + if (pReader->pBlockIdx && pReader->pBlockL) { + } else if (pReader->pBlockIdx) { + // may have problem (todo) + while (pReader->iBlock < pReader->mBlock.nItem) { + SBlock block; + tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, &block, tGetBlock); + + if (block.minVer <= pReader->ever && block.maxVer >= pReader->sver) { + // load data (todo) + } + + // next + pReader->iBlock++; + + if (*ppData) goto _exit; } + } else if (pReader->pBlockL) { + while (pReader->pBlockL) { + if (pReader->pBlockL->minVer <= pReader->ever && pReader->pBlockL->maxVer >= pReader->sver) { + // load data (todo) + } - pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx); - pReader->iBlockIdx++; + // next + pReader->iBlockL++; + if (pReader->iBlockL < taosArrayGetSize(pReader->aBlockL)) { + pReader->pBlockL = (SBlockL*)taosArrayGetSize(pReader->aBlockL); + } else { + pReader->pBlockL = NULL; + } - code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock); - if (code) goto _err; - - pReader->iBlock = 0; + if (*ppData) goto _exit; + } + } else { + tsdbDataFReaderClose(&pReader->pDataFReader); + break; } - - SBlock block; - SBlock* pBlock = █ - while (true) { - if (pReader->iBlock >= pReader->mBlock.nItem) { - pReader->pBlockIdx = NULL; - break; - } - - tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, pBlock, tGetBlock); - pReader->iBlock++; - - if (pBlock->minVersion > pReader->ever || pBlock->maxVersion < pReader->sver) continue; - - code = tsdbReadBlockData(pReader->pDataFReader, pReader->pBlockIdx, pBlock, &pReader->oBlockData, NULL, NULL); - if (code) goto _err; - - // filter - tBlockDataReset(&pReader->nBlockData); - for (int32_t iColData = 0; iColData < taosArrayGetSize(pReader->oBlockData.aIdx); iColData++) { - SColData* pColDataO = tBlockDataGetColDataByIdx(&pReader->oBlockData, iColData); - SColData* pColDataN = NULL; - - code = tBlockDataAddColData(&pReader->nBlockData, taosArrayGetSize(pReader->nBlockData.aIdx), &pColDataN); - if (code) goto _err; - - tColDataInit(pColDataN, pColDataO->cid, pColDataO->type, pColDataO->smaOn); - } - - for (int32_t iRow = 0; iRow < pReader->oBlockData.nRow; iRow++) { - TSDBROW row = tsdbRowFromBlockData(&pReader->oBlockData, iRow); - int64_t version = TSDBROW_VERSION(&row); - - tsdbTrace("vgId:%d, vnode snapshot tsdb read for %s, %" PRId64 "(%" PRId64 " , %" PRId64 ")", - TD_VID(pReader->pTsdb->pVnode), pReader->pTsdb->path, version, pReader->sver, pReader->ever); - - if (version < pReader->sver || version > pReader->ever) continue; - - code = tBlockDataAppendRow(&pReader->nBlockData, &row, NULL); - if (code) goto _err; - } - - if (pReader->nBlockData.nRow <= 0) { - continue; - } - - // org data - // compress data (todo) - int32_t size = sizeof(TABLEID) + tPutBlockData(NULL, &pReader->nBlockData); - - *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size); - if (*ppData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - - SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData); - pHdr->type = pReader->type; - pHdr->size = size; - - TABLEID* pId = (TABLEID*)(&pHdr[1]); - pId->suid = pReader->pBlockIdx->suid; - pId->uid = pReader->pBlockIdx->uid; - - tPutBlockData((uint8_t*)(&pId[1]), &pReader->nBlockData); - - // tsdbInfo("vgId:%d, vnode snapshot read data for %s, fid:%d suid:%" PRId64 " uid:%" PRId64 - // " iBlock:%d minVersion:%d maxVersion:%d nRow:%d out of %d size:%d", - // TD_VID(pTsdb->pVnode), pTsdb->path, pReader->fid, pReader->pBlockIdx->suid, pReader->pBlockIdx->uid, - // pReader->iBlock - 1, pBlock->minVersion, pBlock->maxVersion, pReader->nBlockData.nRow, pBlock->nRow, - // size); - - goto _exit; - } -#endif } } From f59bc9c0a75931155e2262c2f495e35418494983 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 11:02:33 +0800 Subject: [PATCH 64/92] more work --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 33 ++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index dbf710bb20..a970b0be74 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -30,6 +30,8 @@ struct STsdbSnapReader { SArray* aBlockL; // SArray SBlockIdx* pBlockIdx; SBlockL* pBlockL; + SBlock* pBlock; + SBlock block; int32_t iBlockIdx; int32_t iBlockL; @@ -58,7 +60,7 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { pReader->fid = pSet->fid; // load - code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, pSet); + code = tsdbDataFReaderOpen(&pReader->pDataFReader, pTsdb, pSet); if (code) goto _err; code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx); @@ -69,8 +71,25 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { // init pReader->iBlockIdx = 0; - pReader->pBlockIdx = NULL; + if (pReader->iBlockIdx < taosArrayGetSize(pReader->aBlockIdx)) { + pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx); + + code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock); + if (code) goto _err; + + pReader->iBlock = 0; + if (pReader->iBlock < pReader->mBlock.nItem) { + tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, &pReader->block, tGetBlock); + } + } else { + pReader->pBlockIdx = NULL; + } pReader->iBlockL = 0; + if (pReader->iBlockL < taosArrayGetSize(pReader->aBlockL)) { + pReader->pBlockL = (SBlockL*)taosArrayGet(pReader->aBlockL, pReader->iBlockL); + } else { + pReader->pBlockL = NULL; + } tsdbInfo("vgId:%d, vnode snapshot tsdb open data file to read for %s, fid:%d", TD_VID(pTsdb->pVnode), pTsdb->path, pReader->fid); @@ -78,6 +97,16 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { while (true) { if (pReader->pBlockIdx && pReader->pBlockL) { + TABLEID minId = {.suid = pReader->pBlockL->suid, .uid = pReader->pBlockL->minUid}; + TABLEID maxId = {.suid = pReader->pBlockL->suid, .uid = pReader->pBlockL->maxUid}; + + if (tTABLEIDCmprFn(pReader->pBlockIdx, &minId) < 0) { + // TODO + } else if (tTABLEIDCmprFn(pReader->pBlockIdx, &maxId) < 0) { + // TODO + } else { + // TODO + } } else if (pReader->pBlockIdx) { // may have problem (todo) while (pReader->iBlock < pReader->mBlock.nItem) { From c7158f782ebfdf4771699fd6501fcb24e12d6271 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 11:52:43 +0800 Subject: [PATCH 65/92] adjust more --- source/dnode/vnode/src/inc/meta.h | 1 - source/dnode/vnode/src/meta/metaQuery.c | 4 ++- source/dnode/vnode/src/tsdb/tsdbFile.c | 15 ++++++-- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 42 +++++++++++++--------- 4 files changed, 41 insertions(+), 21 deletions(-) diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 2efc33a8ee..adfbb91920 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -66,7 +66,6 @@ int32_t metaCacheOpen(SMeta* pMeta); void metaCacheClose(SMeta* pMeta); int32_t metaCacheUpsert(SMeta* pMeta, SMetaInfo* pInfo); int32_t metaCacheDrop(SMeta* pMeta, int64_t uid); -int32_t metaCacheGet(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo); struct SMeta { TdThreadRwlock lock; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 9c879bf369..9bbff6c6bd 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -127,7 +127,7 @@ _err: // return 0; // } -bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid) { +bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid) { // query uid.idx if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), NULL, NULL) < 0) { return false; @@ -1051,6 +1051,8 @@ int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj return TSDB_CODE_SUCCESS; } +int32_t metaCacheGet(SMeta *pMeta, int64_t uid, SMetaInfo *pInfo); + int32_t metaGetInfo(SMeta *pMeta, int64_t uid, SMetaInfo *pInfo) { int32_t code = 0; void *pData = NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 5db02fa285..00d2ac848f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -188,11 +188,16 @@ int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) { n += tPutI32v(p ? p + n : p, pSet->diskId.level); n += tPutI32v(p ? p + n : p, pSet->diskId.id); n += tPutI32v(p ? p + n : p, pSet->fid); + + // data n += tPutHeadFile(p ? p + n : p, pSet->pHeadF); n += tPutDataFile(p ? p + n : p, pSet->pDataF); - n += tPutLastFile(p ? p + n : p, pSet->pLastF); n += tPutSmaFile(p ? p + n : p, pSet->pSmaF); + // last + n += tPutU8(p ? p + n : p, 1); // for future compatibility + n += tPutLastFile(p ? p + n : p, pSet->pLastF); + return n; } @@ -202,11 +207,17 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) { n += tGetI32v(p + n, &pSet->diskId.level); n += tGetI32v(p + n, &pSet->diskId.id); n += tGetI32v(p + n, &pSet->fid); + + // data n += tGetHeadFile(p + n, pSet->pHeadF); n += tGetDataFile(p + n, pSet->pDataF); - n += tGetLastFile(p + n, pSet->pLastF); n += tGetSmaFile(p + n, pSet->pSmaF); + // last + uint8_t nLast; + n += tGetU8(p + n, &nLast); + n += tGetLastFile(p + n, pSet->pLastF); + return n; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index a970b0be74..744f16b113 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -30,8 +30,6 @@ struct STsdbSnapReader { SArray* aBlockL; // SArray SBlockIdx* pBlockIdx; SBlockL* pBlockL; - SBlock* pBlock; - SBlock block; int32_t iBlockIdx; int32_t iBlockL; @@ -78,9 +76,6 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { if (code) goto _err; pReader->iBlock = 0; - if (pReader->iBlock < pReader->mBlock.nItem) { - tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, &pReader->block, tGetBlock); - } } else { pReader->pBlockIdx = NULL; } @@ -97,18 +92,16 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { while (true) { if (pReader->pBlockIdx && pReader->pBlockL) { - TABLEID minId = {.suid = pReader->pBlockL->suid, .uid = pReader->pBlockL->minUid}; - TABLEID maxId = {.suid = pReader->pBlockL->suid, .uid = pReader->pBlockL->maxUid}; + TABLEID id = {.suid = pReader->pBlockL->suid, .uid = pReader->pBlockL->minUid}; - if (tTABLEIDCmprFn(pReader->pBlockIdx, &minId) < 0) { - // TODO - } else if (tTABLEIDCmprFn(pReader->pBlockIdx, &maxId) < 0) { - // TODO - } else { - // TODO - } + // if (tTABLEIDCmprFn(pReader->pBlockIdx, &minId) < 0) { + // // TODO + // } else if (tTABLEIDCmprFn(pReader->pBlockIdx, &maxId) < 0) { + // // TODO + // } else { + // // TODO + // } } else if (pReader->pBlockIdx) { - // may have problem (todo) while (pReader->iBlock < pReader->mBlock.nItem) { SBlock block; tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, &block, tGetBlock); @@ -119,9 +112,24 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { // next pReader->iBlock++; - - if (*ppData) goto _exit; + if (*ppData) break; } + + if (pReader->iBlock >= pReader->mBlock.nItem) { + pReader->iBlockIdx++; + if (pReader->iBlockIdx < taosArrayGetSize(pReader->aBlockIdx)) { + pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx); + + code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock); + if (code) goto _err; + + pReader->iBlock = 0; + } else { + pReader->pBlockIdx = NULL; + } + } + + if (*ppData) goto _exit; } else if (pReader->pBlockL) { while (pReader->pBlockL) { if (pReader->pBlockL->minVer <= pReader->ever && pReader->pBlockL->maxVer >= pReader->sver) { From 72471af0ce2c8d110c0e47684e5de89e8cd68ab8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 14:26:37 +0800 Subject: [PATCH 66/92] adjust more code --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 87 ++++++++++++++++++++++++-- 1 file changed, 82 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index ad759f1ef8..70d723cb3d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1850,6 +1850,8 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol uint8_t **ppBuf) { int32_t code = 0; + ASSERT(pColData->flag && (pColData->flag != HAS_NONE) && (pColData->flag != HAS_NULL)); + pBlockCol->szBitmap = 0; pBlockCol->szOffset = 0; pBlockCol->szValue = 0; @@ -1857,9 +1859,47 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol int32_t size = 0; // bitmap if (pColData->flag != HAS_VALUE) { - code = tsdbCmprData(pColData->pBitMap, BIT2_SIZE(pColData->nVal), TSDB_DATA_TYPE_TINYINT, cmprAlg, ppOut, - nOut + size, &pBlockCol->szBitmap, ppBuf); + uint8_t *pBitMap = pColData->pBitMap; + int32_t szBitMap = BIT2_SIZE(pColData->nVal); + + // BIT2 to BIT1 + if (pColData->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) { + szBitMap = BIT1_SIZE(pColData->nVal); + pBitMap = taosMemoryCalloc(1, szBitMap); + if (pBitMap == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + uint8_t v = GET_BIT2(pColData->pBitMap, iVal); + switch (pColData->flag) { + case (HAS_NULL | HAS_NONE): + SET_BIT1(pBitMap, iVal, v); + break; + case (HAS_VALUE | HAS_NONE): + if (v) { + SET_BIT1(pBitMap, iVal, 1); + } else { + SET_BIT1(pBitMap, iVal, 0); + } + break; + case (HAS_VALUE | HAS_NULL): + SET_BIT1(pBitMap, iVal, v - 1); + break; + default: + ASSERT(0); + } + } + } + + code = tsdbCmprData(pBitMap, szBitMap, TSDB_DATA_TYPE_TINYINT, cmprAlg, ppOut, nOut + size, &pBlockCol->szBitmap, + ppBuf); if (code) goto _exit; + + if (pColData->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) { + taosMemoryFree(pBitMap); + } } size += pBlockCol->szBitmap; @@ -1909,9 +1949,46 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in uint8_t *p = pIn; // bitmap if (pBlockCol->szBitmap) { - code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pColData->pBitMap, - BIT2_SIZE(pColData->nVal), ppBuf); - if (code) goto _exit; + if (pBlockCol->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) { + uint8_t *pBitMap = NULL; + code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pBitMap, + BIT1_SIZE(pColData->nVal), ppBuf); + if (code) goto _exit; + + code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal)); + if (code) { + tFree(pBitMap); + goto _exit; + } + + // BIT1 to BIT2 + for (int32_t iVal = 0; iVal < nVal; iVal++) { + uint8_t v = GET_BIT1(pBitMap, iVal); + switch (pBlockCol->flag) { + case (HAS_NULL | HAS_NONE): + SET_BIT2(pColData->pBitMap, iVal, v); + break; + case (HAS_VALUE | HAS_NONE): + if (v) { + SET_BIT2(pColData->pBitMap, iVal, 2); + } else { + SET_BIT2(pColData->pBitMap, iVal, 0); + } + break; + case (HAS_VALUE | HAS_NULL): + SET_BIT2(pColData->pBitMap, iVal, v + 1); + break; + default: + ASSERT(0); + } + } + + tFree(pBitMap); + } else { + code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pColData->pBitMap, + BIT2_SIZE(pColData->nVal), ppBuf); + if (code) goto _exit; + } } p += pBlockCol->szBitmap; From 189469b71161159df846bdae00543c926c4cfeaf Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 15:30:54 +0800 Subject: [PATCH 67/92] fix more --- include/common/tdataformat.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index b9ce7368a7..af7c88acde 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -43,11 +43,12 @@ const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, {0b00000000, 0b00010000, 0b00100000, 4}, {0b00000000, 0b01000000, 0b10000000, 6}}; +#define N1(n) ((((uint8_t)1) << (n)) - 1) #define BIT1_SIZE(n) ((((n)-1) >> 3) + 1) #define BIT2_SIZE(n) ((((n)-1) >> 2) + 1) -#define SET_BIT1(p, i, v) ((p)[(i) >> 3] |= (((uint8_t)v) << ((i)&7))) +#define SET_BIT1(p, i, v) ((p)[(i) >> 3] = (p)[(i) >> 3] & N1((i)&7) | (((uint8_t)(v)) << ((i)&7))) #define GET_BIT1(p, i) (((p)[(i) >> 3] >> ((i)&7)) & ((uint8_t)1)) -#define SET_BIT2(p, i, v) ((p)[(i) >> 2] |= BIT2_MAP[(i)&3][v]) +#define SET_BIT2(p, i, v) ((p)[(i) >> 2] = (p)[(i) >> 2] & N1(BIT2_MAP[(i)&3][3]) | BIT2_MAP[(i)&3][(v)]) #define GET_BIT2(p, i) (((p)[(i) >> 2] >> BIT2_MAP[(i)&3][3]) & ((uint8_t)3)) // STSchema From 029c16e487216fce3d3ea9fa21f6a19980449e95 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 17:13:39 +0800 Subject: [PATCH 68/92] more code --- source/dnode/vnode/src/inc/tsdb.h | 8 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 - source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 2 + source/dnode/vnode/src/tsdb/tsdbUtil.c | 140 +++++++++++++----- 4 files changed, 106 insertions(+), 46 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 0c706c3b6b..9e02453ccf 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -65,6 +65,7 @@ typedef struct SBlockInfo SBlockInfo; typedef struct SSmaInfo SSmaInfo; typedef struct SBlockCol SBlockCol; +#define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) #define TSDB_MAX_SUBBLOCKS 8 #define TSDB_FHDR_SIZE 512 @@ -150,12 +151,7 @@ SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx); void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData); int32_t tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest); int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData); - -#if 1 -int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); -int32_t tPutBlockData(uint8_t *p, SBlockData *pBlockData); -int32_t tGetBlockData(uint8_t *p, SBlockData *pBlockData); -#endif +int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); // SDiskDataHdr int32_t tPutDiskDataHdr(uint8_t *p, void *ph); int32_t tGetDiskDataHdr(uint8_t *p, void *ph); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 55ef576916..dc9cf19593 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -15,8 +15,6 @@ #include "tsdb.h" -#define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) - // SDelFWriter ==================================================== int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb) { int32_t code = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 744f16b113..ce970ea870 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -94,6 +94,8 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { if (pReader->pBlockIdx && pReader->pBlockL) { TABLEID id = {.suid = pReader->pBlockL->suid, .uid = pReader->pBlockL->minUid}; + ASSERT(0); + // if (tTABLEIDCmprFn(pReader->pBlockIdx, &minId) < 0) { // // TODO // } else if (tTABLEIDCmprFn(pReader->pBlockIdx, &maxId) < 0) { diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 70d723cb3d..740da45393 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1500,50 +1500,114 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColD *ppColData = NULL; } -int32_t tPutBlockData(uint8_t *p, SBlockData *pBlockData) { - int32_t n = 0; +int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut) { + int32_t code = 0; - n += tPutI32v(p ? p + n : p, pBlockData->nRow); - if (p) { - memcpy(p + n, pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow); - } - n = n + sizeof(int64_t) * pBlockData->nRow; - if (p) { - memcpy(p + n, pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow); - } - n = n + sizeof(TSKEY) * pBlockData->nRow; + SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, + .fmtVer = 0, + .suid = pBlockData->suid, + .uid = pBlockData->uid, + .nRow = pBlockData->nRow, + .cmprAlg = cmprAlg}; - int32_t nCol = taosArrayGetSize(pBlockData->aIdx); - n += tPutI32v(p ? p + n : p, nCol); - for (int32_t iCol = 0; iCol < nCol; iCol++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iCol); - n += tPutColData(p ? p + n : p, pColData); + uint8_t *pBuf1 = NULL; + uint8_t *pBuf2 = NULL; + uint8_t *pBuf3 = NULL; + uint8_t *pBuf4 = NULL; + + // encode ================= + // columns AND SBlockCol + int32_t nBuf1 = 0; + for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { + SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); + + ASSERT(pColData->flag); + + if (pColData->flag == HAS_NONE) continue; + + SBlockCol blockCol = {.cid = pColData->cid, + .type = pColData->type, + .smaOn = pColData->smaOn, + .flag = pColData->flag, + .szOrigin = pColData->nData}; + + if (pColData->flag != HAS_NULL) { + code = tsdbCmprColData(pColData, cmprAlg, &blockCol, &pBuf1, nBuf1, &pBuf3); + if (code) goto _exit; + + blockCol.offset = nBuf1; + nBuf1 = nBuf1 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); + } + + code = tRealloc(&pBuf2, hdr.szBlkCol + tPutBlockCol(NULL, &blockCol)); + if (code) goto _exit; + hdr.szBlkCol += tPutBlockCol(pBuf2 + hdr.szBlkCol, &blockCol); } - return n; + int32_t nBuf2 = 0; + if (hdr.szBlkCol > 0) { + nBuf2 = hdr.szBlkCol + sizeof(TSCKSUM); + + code = tRealloc(&pBuf2, nBuf2); + if (code) goto _exit; + + taosCalcChecksumAppend(0, pBuf2, nBuf2); + } + + // uid + version + tskey + int32_t nBuf3 = 0; + if (pBlockData->uid == 0) { + code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, + &pBuf3, nBuf3, &hdr.szUid, &pBuf4); + if (code) goto _exit; + } + nBuf3 += hdr.szUid; + + code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, + cmprAlg, &pBuf3, nBuf3, &hdr.szVer, &pBuf4); + if (code) goto _exit; + nBuf3 += hdr.szVer; + + code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, + cmprAlg, &pBuf3, nBuf3, &hdr.szKey, &pBuf4); + if (code) goto _exit; + nBuf3 += hdr.szKey; + + nBuf3 += sizeof(TSCKSUM); + code = tRealloc(&pBuf3, nBuf3); + if (code) goto _exit; + + // hdr + int32_t nBuf4 = tPutDiskDataHdr(NULL, &hdr); + code = tRealloc(&pBuf4, nBuf4); + if (code) goto _exit; + tPutDiskDataHdr(pBuf4, &hdr); + taosCalcChecksumAppend(taosCalcChecksum(0, pBuf4, nBuf4), pBuf3, nBuf3); + + // aggragate + if (ppOut) { + *szOut = nBuf1 + nBuf2 + nBuf3 + nBuf4; + code = tRealloc(ppOut, *szOut); + if (code) goto _exit; + + memcpy(*ppOut, pBuf4, nBuf4); + memcpy(*ppOut + nBuf4, pBuf3, nBuf3); + if (nBuf2) { + memcpy(*ppOut + nBuf4 + nBuf3, pBuf2, nBuf2); + } + if (nBuf1) { + memcpy(*ppOut + nBuf4 + nBuf3 + nBuf2, pBuf1, nBuf1); + } + } + +_exit: + return code; } -int32_t tGetBlockData(uint8_t *p, SBlockData *pBlockData) { - int32_t n = 0; - - tBlockDataReset(pBlockData); - - n += tGetI32v(p + n, &pBlockData->nRow); - pBlockData->aVersion = (int64_t *)(p + n); - n = n + sizeof(int64_t) * pBlockData->nRow; - pBlockData->aTSKEY = (TSKEY *)(p + n); - n = n + sizeof(TSKEY) * pBlockData->nRow; - - int32_t nCol; - n += tGetI32v(p + n, &nCol); - for (int32_t iCol = 0; iCol < nCol; iCol++) { - SColData *pColData; - - if (tBlockDataAddColData(pBlockData, iCol, &pColData)) return -1; - n += tGetColData(p + n, pColData); - } - - return n; +int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData) { + int32_t code = 0; + // TODO + return code; } // SDiskDataHdr ============================== From d3d03d8c4d9ae6914d19e339dab0cd6c0d466636 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 17:45:52 +0800 Subject: [PATCH 69/92] adjust more code --- source/dnode/vnode/src/inc/tsdb.h | 7 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 198 +++++++++--------- 2 files changed, 102 insertions(+), 103 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 9e02453ccf..f31a373254 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -574,7 +574,7 @@ struct SDelFWriter { SDelFile fDel; TdFilePtr pWriteH; - uint8_t *pBuf1; + uint8_t *aBuf[1]; }; struct SDataFWriter { @@ -591,10 +591,7 @@ struct SDataFWriter { SLastFile fLast; SSmaFile fSma; - uint8_t *pBuf1; - uint8_t *pBuf2; - uint8_t *pBuf3; - uint8_t *pBuf4; + uint8_t *aBuf[4]; }; struct STsdbReadSnap { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index dc9cf19593..f3911350e6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -75,7 +75,9 @@ int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync) { goto _err; } - tFree(pWriter->pBuf1); + for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t *); iBuf++) { + tFree(pWriter->aBuf[iBuf]); + } taosMemoryFree(pWriter); *ppWriter = NULL; @@ -99,21 +101,21 @@ int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelId size += sizeof(TSCKSUM); // alloc - code = tRealloc(&pWriter->pBuf1, size); + code = tRealloc(&pWriter->aBuf[0], size); if (code) goto _err; // build n = 0; - n += tPutU32(pWriter->pBuf1 + n, TSDB_FILE_DLMT); + n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { - n += tPutDelData(pWriter->pBuf1 + n, taosArrayGet(aDelData, iDelData)); + n += tPutDelData(pWriter->aBuf[0] + n, taosArrayGet(aDelData, iDelData)); } - taosCalcChecksumAppend(0, pWriter->pBuf1, size); + taosCalcChecksumAppend(0, pWriter->aBuf[0], size); ASSERT(n + sizeof(TSCKSUM) == size); // write - n = taosWriteFile(pWriter->pWriteH, pWriter->pBuf1, size); + n = taosWriteFile(pWriter->pWriteH, pWriter->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -147,21 +149,21 @@ int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx) { size += sizeof(TSCKSUM); // alloc - code = tRealloc(&pWriter->pBuf1, size); + code = tRealloc(&pWriter->aBuf[0], size); if (code) goto _err; // build n = 0; - n += tPutU32(pWriter->pBuf1 + n, TSDB_FILE_DLMT); + n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { - n += tPutDelIdx(pWriter->pBuf1 + n, taosArrayGet(aDelIdx, iDelIdx)); + n += tPutDelIdx(pWriter->aBuf[0] + n, taosArrayGet(aDelIdx, iDelIdx)); } - taosCalcChecksumAppend(0, pWriter->pBuf1, size); + taosCalcChecksumAppend(0, pWriter->aBuf[0], size); ASSERT(n + sizeof(TSCKSUM) == size); // write - n = taosWriteFile(pWriter->pWriteH, pWriter->pBuf1, size); + n = taosWriteFile(pWriter->pWriteH, pWriter->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -215,7 +217,7 @@ struct SDelFReader { SDelFile fDel; TdFilePtr pReadH; - uint8_t *pBuf1; + uint8_t *aBuf[1]; }; int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb) { @@ -262,7 +264,9 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader) { code = TAOS_SYSTEM_ERROR(errno); goto _exit; } - tFree(pReader->pBuf1); + for (int32_t iBuf = 0; iBuf < sizeof(pReader->aBuf) / sizeof(uint8_t *); iBuf++) { + tFree(pReader->aBuf[iBuf]); + } taosMemoryFree(pReader); } *ppReader = NULL; @@ -286,11 +290,11 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData } // alloc - code = tRealloc(&pReader->pBuf1, size); + code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; // read - n = taosReadFile(pReader->pReadH, pReader->pBuf1, size); + n = taosReadFile(pReader->pReadH, pReader->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -300,7 +304,7 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData } // check - if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { + if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } @@ -309,10 +313,10 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData n = 0; uint32_t delimiter; - n += tGetU32(pReader->pBuf1 + n, &delimiter); + n += tGetU32(pReader->aBuf[0] + n, &delimiter); while (n < size - sizeof(TSCKSUM)) { SDelData delData; - n += tGetDelData(pReader->pBuf1 + n, &delData); + n += tGetDelData(pReader->aBuf[0] + n, &delData); if (taosArrayPush(aDelData, &delData) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -344,11 +348,11 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { } // alloc - code = tRealloc(&pReader->pBuf1, size); + code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; // read - n = taosReadFile(pReader->pReadH, pReader->pBuf1, size); + n = taosReadFile(pReader->pReadH, pReader->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -358,7 +362,7 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { } // check - if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { + if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } @@ -366,13 +370,13 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { // decode n = 0; uint32_t delimiter; - n += tGetU32(pReader->pBuf1 + n, &delimiter); + n += tGetU32(pReader->aBuf[0] + n, &delimiter); ASSERT(delimiter == TSDB_FILE_DLMT); while (n < size - sizeof(TSCKSUM)) { SDelIdx delIdx; - n += tGetDelIdx(pReader->pBuf1 + n, &delIdx); + n += tGetDelIdx(pReader->aBuf[0] + n, &delIdx); if (taosArrayPush(aDelIdx, &delIdx) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -398,9 +402,7 @@ struct SDataFReader { TdFilePtr pLastFD; TdFilePtr pSmaFD; - uint8_t *pBuf1; - uint8_t *pBuf2; - uint8_t *pBuf3; + uint8_t *aBuf[3]; }; int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) { @@ -483,9 +485,10 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { goto _err; } - tFree((*ppReader)->pBuf1); - tFree((*ppReader)->pBuf2); - tFree((*ppReader)->pBuf3); + for (int32_t iBuf = 0; iBuf < sizeof((*ppReader)->aBuf) / sizeof(uint8_t *); iBuf++) { + tFree((*ppReader)->aBuf[iBuf]); + } + taosMemoryFree(*ppReader); _exit: @@ -510,7 +513,7 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { } // alloc - code = tRealloc(&pReader->pBuf1, size); + code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; // seek @@ -520,7 +523,7 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { } // read - n = taosReadFile(pReader->pHeadFD, pReader->pBuf1, size); + n = taosReadFile(pReader->pHeadFD, pReader->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -530,19 +533,19 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { } // check - if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { + if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } // decode n = 0; - n = tGetU32(pReader->pBuf1 + n, &delimiter); + n = tGetU32(pReader->aBuf[0] + n, &delimiter); ASSERT(delimiter == TSDB_FILE_DLMT); while (n < size - sizeof(TSCKSUM)) { SBlockIdx blockIdx; - n += tGetBlockIdx(pReader->pBuf1 + n, &blockIdx); + n += tGetBlockIdx(pReader->aBuf[0] + n, &blockIdx); if (taosArrayPush(aBlockIdx, &blockIdx) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -573,7 +576,7 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL) { } // alloc - code = tRealloc(&pReader->pBuf1, size); + code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; // seek @@ -583,7 +586,7 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL) { } // read - n = taosReadFile(pReader->pLastFD, pReader->pBuf1, size); + n = taosReadFile(pReader->pLastFD, pReader->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -593,19 +596,19 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL) { } // check - if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { + if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } // decode n = 0; - n = tGetU32(pReader->pBuf1 + n, &delimiter); + n = tGetU32(pReader->aBuf[0] + n, &delimiter); ASSERT(delimiter == TSDB_FILE_DLMT); while (n < size - sizeof(TSCKSUM)) { SBlockL blockl; - n += tGetBlockL(pReader->pBuf1 + n, &blockl); + n += tGetBlockL(pReader->aBuf[0] + n, &blockl); if (taosArrayPush(aBlockL, &blockl) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -631,7 +634,7 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBl int64_t tn; // alloc - code = tRealloc(&pReader->pBuf1, size); + code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; // seek @@ -641,7 +644,7 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBl } // read - n = taosReadFile(pReader->pHeadFD, pReader->pBuf1, size); + n = taosReadFile(pReader->pHeadFD, pReader->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -651,7 +654,7 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBl } // check - if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { + if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } @@ -660,10 +663,10 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBl n = 0; uint32_t delimiter; - n += tGetU32(pReader->pBuf1 + n, &delimiter); + n += tGetU32(pReader->aBuf[0] + n, &delimiter); ASSERT(delimiter == TSDB_FILE_DLMT); - tn = tGetMapData(pReader->pBuf1 + n, mBlock); + tn = tGetMapData(pReader->aBuf[0] + n, mBlock); if (tn < 0) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -688,11 +691,11 @@ int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnD // alloc int32_t size = pSmaInfo->size + sizeof(TSCKSUM); - code = tRealloc(&pReader->pBuf1, size); + code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; // read - int64_t n = taosReadFile(pReader->pSmaFD, pReader->pBuf1, size); + int64_t n = taosReadFile(pReader->pSmaFD, pReader->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -702,7 +705,7 @@ int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnD } // check - if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { + if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } @@ -712,7 +715,7 @@ int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnD while (n < pSmaInfo->size) { SColumnDataAgg sma; - n += tGetColumnDataAgg(pReader->pBuf1 + n, &sma); + n += tGetColumnDataAgg(pReader->aBuf[0] + n, &sma); if (taosArrayPush(aColumnDataAgg, &sma) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -735,10 +738,10 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo TdFilePtr pFD = fromLast ? pReader->pLastFD : pReader->pDataFD; // uid + version + tskey - code = tsdbReadAndCheck(pFD, pBlkInfo->offset, &pReader->pBuf1, pBlkInfo->szKey, 1); + code = tsdbReadAndCheck(pFD, pBlkInfo->offset, &pReader->aBuf[0], pBlkInfo->szKey, 1); if (code) goto _err; SDiskDataHdr hdr; - uint8_t *p = pReader->pBuf1 + tGetDiskDataHdr(pReader->pBuf1, &hdr); + uint8_t *p = pReader->aBuf[0] + tGetDiskDataHdr(pReader->aBuf[0], &hdr); ASSERT(hdr.delimiter == TSDB_FILE_DLMT); ASSERT(pBlockData->suid == hdr.suid); @@ -750,7 +753,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo if (hdr.uid == 0) { ASSERT(hdr.szUid); code = tsdbDecmprData(p, hdr.szUid, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aUid, - sizeof(int64_t) * hdr.nRow, &pReader->pBuf2); + sizeof(int64_t) * hdr.nRow, &pReader->aBuf[1]); if (code) goto _err; } else { ASSERT(!hdr.szUid); @@ -759,24 +762,24 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo // version code = tsdbDecmprData(p, hdr.szVer, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aVersion, - sizeof(int64_t) * hdr.nRow, &pReader->pBuf2); + sizeof(int64_t) * hdr.nRow, &pReader->aBuf[1]); if (code) goto _err; p += hdr.szVer; // TSKEY code = tsdbDecmprData(p, hdr.szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr.cmprAlg, (uint8_t **)&pBlockData->aTSKEY, - sizeof(TSKEY) * hdr.nRow, &pReader->pBuf2); + sizeof(TSKEY) * hdr.nRow, &pReader->aBuf[1]); if (code) goto _err; p += hdr.szKey; - ASSERT(p - pReader->pBuf1 == pBlkInfo->szKey - sizeof(TSCKSUM)); + ASSERT(p - pReader->aBuf[0] == pBlkInfo->szKey - sizeof(TSCKSUM)); // read and decode columns if (taosArrayGetSize(pBlockData->aIdx) == 0) goto _exit; if (hdr.szBlkCol > 0) { int64_t offset = pBlkInfo->offset + pBlkInfo->szKey; - code = tsdbReadAndCheck(pFD, offset, &pReader->pBuf1, hdr.szBlkCol + sizeof(TSCKSUM), 1); + code = tsdbReadAndCheck(pFD, offset, &pReader->aBuf[0], hdr.szBlkCol + sizeof(TSCKSUM), 1); if (code) goto _err; } @@ -789,7 +792,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo while (pBlockCol && pBlockCol->cid < pColData->cid) { if (n < hdr.szBlkCol) { - n += tGetBlockCol(pReader->pBuf1 + n, pBlockCol); + n += tGetBlockCol(pReader->aBuf[0] + n, pBlockCol); } else { ASSERT(n == hdr.szBlkCol); pBlockCol = NULL; @@ -817,10 +820,10 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo int64_t offset = pBlkInfo->offset + pBlkInfo->szKey + hdr.szBlkCol + sizeof(TSCKSUM) + pBlockCol->offset; int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM); - code = tsdbReadAndCheck(pFD, offset, &pReader->pBuf2, size, 0); + code = tsdbReadAndCheck(pFD, offset, &pReader->aBuf[1], size, 0); if (code) goto _err; - code = tsdbDecmprColData(pReader->pBuf2, pBlockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->pBuf3); + code = tsdbDecmprColData(pReader->aBuf[1], pBlockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->aBuf[2]); if (code) goto _err; } } @@ -1096,10 +1099,9 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { goto _err; } - tFree((*ppWriter)->pBuf1); - tFree((*ppWriter)->pBuf2); - tFree((*ppWriter)->pBuf3); - tFree((*ppWriter)->pBuf4); + for (int32_t iBuf = 0; iBuf < sizeof((*ppWriter)->aBuf) / sizeof(uint8_t *); iBuf++) { + tFree((*ppWriter)->aBuf[iBuf]); + } taosMemoryFree(*ppWriter); _exit: *ppWriter = NULL; @@ -1210,21 +1212,21 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) { size += sizeof(TSCKSUM); // alloc - code = tRealloc(&pWriter->pBuf1, size); + code = tRealloc(&pWriter->aBuf[0], size); if (code) goto _err; // build n = 0; - n = tPutU32(pWriter->pBuf1 + n, TSDB_FILE_DLMT); + n = tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) { - n += tPutBlockIdx(pWriter->pBuf1 + n, taosArrayGet(aBlockIdx, iBlockIdx)); + n += tPutBlockIdx(pWriter->aBuf[0] + n, taosArrayGet(aBlockIdx, iBlockIdx)); } - taosCalcChecksumAppend(0, pWriter->pBuf1, size); + taosCalcChecksumAppend(0, pWriter->aBuf[0], size); ASSERT(n + sizeof(TSCKSUM) == size); // write - n = taosWriteFile(pWriter->pHeadFD, pWriter->pBuf1, size); + n = taosWriteFile(pWriter->pHeadFD, pWriter->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1254,19 +1256,19 @@ int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, SBlockIdx *pBloc // alloc size = sizeof(uint32_t) + tPutMapData(NULL, mBlock) + sizeof(TSCKSUM); - code = tRealloc(&pWriter->pBuf1, size); + code = tRealloc(&pWriter->aBuf[0], size); if (code) goto _err; // build n = 0; - n += tPutU32(pWriter->pBuf1 + n, TSDB_FILE_DLMT); - n += tPutMapData(pWriter->pBuf1 + n, mBlock); - taosCalcChecksumAppend(0, pWriter->pBuf1, size); + n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); + n += tPutMapData(pWriter->aBuf[0] + n, mBlock); + taosCalcChecksumAppend(0, pWriter->aBuf[0], size); ASSERT(n + sizeof(TSCKSUM) == size); // write - n = taosWriteFile(pWriter->pHeadFD, pWriter->pBuf1, size); + n = taosWriteFile(pWriter->pHeadFD, pWriter->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1308,21 +1310,21 @@ int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) { size += sizeof(TSCKSUM); // alloc - code = tRealloc(&pWriter->pBuf1, size); + code = tRealloc(&pWriter->aBuf[0], size); if (code) goto _err; // encode n = 0; - n += tPutU32(pWriter->pBuf1 + n, TSDB_FILE_DLMT); + n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aBlockL); iBlockL++) { - n += tPutBlockL(pWriter->pBuf1 + n, taosArrayGet(aBlockL, iBlockL)); + n += tPutBlockL(pWriter->aBuf[0] + n, taosArrayGet(aBlockL, iBlockL)); } - taosCalcChecksumAppend(0, pWriter->pBuf1, size); + taosCalcChecksumAppend(0, pWriter->aBuf[0], size); ASSERT(n + sizeof(TSCKSUM) == size); // write - n = taosWriteFile(pWriter->pLastFD, pWriter->pBuf1, size); + n = taosWriteFile(pWriter->pLastFD, pWriter->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1381,21 +1383,21 @@ static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, SColumnDataAgg sma; tsdbCalcColDataSMA(pColData, &sma); - code = tRealloc(&pWriter->pBuf1, pSmaInfo->size + tPutColumnDataAgg(NULL, &sma)); + code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &sma)); if (code) goto _err; - pSmaInfo->size += tPutColumnDataAgg(pWriter->pBuf1 + pSmaInfo->size, &sma); + pSmaInfo->size += tPutColumnDataAgg(pWriter->aBuf[0] + pSmaInfo->size, &sma); } // write if (pSmaInfo->size) { int32_t size = pSmaInfo->size + sizeof(TSCKSUM); - code = tRealloc(&pWriter->pBuf1, size); + code = tRealloc(&pWriter->aBuf[0], size); if (code) goto _err; - taosCalcChecksumAppend(0, pWriter->pBuf1, size); + taosCalcChecksumAppend(0, pWriter->aBuf[0], size); - int64_t n = taosWriteFile(pWriter->pSmaFD, pWriter->pBuf1, size); + int64_t n = taosWriteFile(pWriter->pSmaFD, pWriter->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1447,57 +1449,57 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock .szOrigin = pColData->nData}; if (pColData->flag != HAS_NULL) { - code = tsdbCmprColData(pColData, cmprAlg, &blockCol, &pWriter->pBuf1, nBuf1, &pWriter->pBuf3); + code = tsdbCmprColData(pColData, cmprAlg, &blockCol, &pWriter->aBuf[0], nBuf1, &pWriter->aBuf[2]); if (code) goto _err; blockCol.offset = nBuf1; nBuf1 = nBuf1 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); } - code = tRealloc(&pWriter->pBuf2, hdr.szBlkCol + tPutBlockCol(NULL, &blockCol)); + code = tRealloc(&pWriter->aBuf[1], hdr.szBlkCol + tPutBlockCol(NULL, &blockCol)); if (code) goto _err; - hdr.szBlkCol += tPutBlockCol(pWriter->pBuf2 + hdr.szBlkCol, &blockCol); + hdr.szBlkCol += tPutBlockCol(pWriter->aBuf[1] + hdr.szBlkCol, &blockCol); } int32_t nBuf2 = 0; if (hdr.szBlkCol > 0) { nBuf2 = hdr.szBlkCol + sizeof(TSCKSUM); - code = tRealloc(&pWriter->pBuf2, nBuf2); + code = tRealloc(&pWriter->aBuf[1], nBuf2); if (code) goto _err; - taosCalcChecksumAppend(0, pWriter->pBuf2, nBuf2); + taosCalcChecksumAppend(0, pWriter->aBuf[1], nBuf2); } // uid + version + tskey int32_t nBuf3 = 0; if (pBlockData->uid == 0) { code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, - &pWriter->pBuf3, nBuf3, &hdr.szUid, &pWriter->pBuf4); + &pWriter->aBuf[2], nBuf3, &hdr.szUid, &pWriter->aBuf[3]); if (code) goto _err; } nBuf3 += hdr.szUid; code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, - cmprAlg, &pWriter->pBuf3, nBuf3, &hdr.szVer, &pWriter->pBuf4); + cmprAlg, &pWriter->aBuf[2], nBuf3, &hdr.szVer, &pWriter->aBuf[3]); if (code) goto _err; nBuf3 += hdr.szVer; code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, - cmprAlg, &pWriter->pBuf3, nBuf3, &hdr.szKey, &pWriter->pBuf4); + cmprAlg, &pWriter->aBuf[2], nBuf3, &hdr.szKey, &pWriter->aBuf[3]); if (code) goto _err; nBuf3 += hdr.szKey; nBuf3 += sizeof(TSCKSUM); - code = tRealloc(&pWriter->pBuf3, nBuf3); + code = tRealloc(&pWriter->aBuf[2], nBuf3); if (code) goto _err; // hdr int32_t nBuf4 = tPutDiskDataHdr(NULL, &hdr); - code = tRealloc(&pWriter->pBuf4, nBuf4); + code = tRealloc(&pWriter->aBuf[3], nBuf4); if (code) goto _err; - tPutDiskDataHdr(pWriter->pBuf4, &hdr); - taosCalcChecksumAppend(taosCalcChecksum(0, pWriter->pBuf4, nBuf4), pWriter->pBuf3, nBuf3); + tPutDiskDataHdr(pWriter->aBuf[3], &hdr); + taosCalcChecksumAppend(taosCalcChecksum(0, pWriter->aBuf[3], nBuf4), pWriter->aBuf[2], nBuf3); // write ================= TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; @@ -1505,20 +1507,20 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock pBlkInfo->szKey = nBuf4 + nBuf3; pBlkInfo->szBlock = nBuf1 + nBuf2 + nBuf3 + nBuf4; - int64_t n = taosWriteFile(pFD, pWriter->pBuf4, nBuf4); + int64_t n = taosWriteFile(pFD, pWriter->aBuf[3], nBuf4); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - n = taosWriteFile(pFD, pWriter->pBuf3, nBuf3); + n = taosWriteFile(pFD, pWriter->aBuf[2], nBuf3); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } if (nBuf2) { - n = taosWriteFile(pFD, pWriter->pBuf2, nBuf2); + n = taosWriteFile(pFD, pWriter->aBuf[1], nBuf2); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1526,7 +1528,7 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock } if (nBuf1) { - n = taosWriteFile(pFD, pWriter->pBuf1, nBuf1); + n = taosWriteFile(pFD, pWriter->aBuf[0], nBuf1); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; From 47cddbfe2b4473a3a8d9ff3d81eb73714a1346b3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 17:52:48 +0800 Subject: [PATCH 70/92] mroe code --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 67 ++++++++++++-------------- 1 file changed, 31 insertions(+), 36 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 740da45393..8355149292 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1500,7 +1500,7 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColD *ppColData = NULL; } -int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut) { +int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[]) { int32_t code = 0; SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, @@ -1510,14 +1510,9 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, .nRow = pBlockData->nRow, .cmprAlg = cmprAlg}; - uint8_t *pBuf1 = NULL; - uint8_t *pBuf2 = NULL; - uint8_t *pBuf3 = NULL; - uint8_t *pBuf4 = NULL; - // encode ================= // columns AND SBlockCol - int32_t nBuf1 = 0; + int32_t nBuf0 = 0; for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); @@ -1532,71 +1527,71 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, .szOrigin = pColData->nData}; if (pColData->flag != HAS_NULL) { - code = tsdbCmprColData(pColData, cmprAlg, &blockCol, &pBuf1, nBuf1, &pBuf3); + code = tsdbCmprColData(pColData, cmprAlg, &blockCol, &aBuf[0], nBuf0, &aBuf[2]); if (code) goto _exit; - blockCol.offset = nBuf1; - nBuf1 = nBuf1 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); + blockCol.offset = nBuf0; + nBuf0 = nBuf0 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); } - code = tRealloc(&pBuf2, hdr.szBlkCol + tPutBlockCol(NULL, &blockCol)); + code = tRealloc(&aBuf[1], hdr.szBlkCol + tPutBlockCol(NULL, &blockCol)); if (code) goto _exit; - hdr.szBlkCol += tPutBlockCol(pBuf2 + hdr.szBlkCol, &blockCol); + hdr.szBlkCol += tPutBlockCol(aBuf[1] + hdr.szBlkCol, &blockCol); } - int32_t nBuf2 = 0; + int32_t nBuf1 = 0; if (hdr.szBlkCol > 0) { - nBuf2 = hdr.szBlkCol + sizeof(TSCKSUM); + nBuf1 = hdr.szBlkCol + sizeof(TSCKSUM); - code = tRealloc(&pBuf2, nBuf2); + code = tRealloc(&aBuf[1], nBuf1); if (code) goto _exit; - taosCalcChecksumAppend(0, pBuf2, nBuf2); + taosCalcChecksumAppend(0, aBuf[1], nBuf1); } // uid + version + tskey - int32_t nBuf3 = 0; + int32_t nBuf2 = 0; if (pBlockData->uid == 0) { code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, - &pBuf3, nBuf3, &hdr.szUid, &pBuf4); + &aBuf[2], nBuf2, &hdr.szUid, &aBuf[3]); if (code) goto _exit; } - nBuf3 += hdr.szUid; + nBuf2 += hdr.szUid; code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, - cmprAlg, &pBuf3, nBuf3, &hdr.szVer, &pBuf4); + cmprAlg, &aBuf[2], nBuf2, &hdr.szVer, &aBuf[3]); if (code) goto _exit; - nBuf3 += hdr.szVer; + nBuf2 += hdr.szVer; code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, - cmprAlg, &pBuf3, nBuf3, &hdr.szKey, &pBuf4); + cmprAlg, &aBuf[2], nBuf2, &hdr.szKey, &aBuf[3]); if (code) goto _exit; - nBuf3 += hdr.szKey; + nBuf2 += hdr.szKey; - nBuf3 += sizeof(TSCKSUM); - code = tRealloc(&pBuf3, nBuf3); + nBuf2 += sizeof(TSCKSUM); + code = tRealloc(&aBuf[2], nBuf2); if (code) goto _exit; // hdr - int32_t nBuf4 = tPutDiskDataHdr(NULL, &hdr); - code = tRealloc(&pBuf4, nBuf4); + int32_t nBuf3 = tPutDiskDataHdr(NULL, &hdr); + code = tRealloc(&aBuf[3], nBuf3); if (code) goto _exit; - tPutDiskDataHdr(pBuf4, &hdr); - taosCalcChecksumAppend(taosCalcChecksum(0, pBuf4, nBuf4), pBuf3, nBuf3); + tPutDiskDataHdr(aBuf[3], &hdr); + taosCalcChecksumAppend(taosCalcChecksum(0, aBuf[3], nBuf3), aBuf[2], nBuf2); // aggragate if (ppOut) { - *szOut = nBuf1 + nBuf2 + nBuf3 + nBuf4; + *szOut = nBuf0 + nBuf1 + nBuf2 + nBuf3; code = tRealloc(ppOut, *szOut); if (code) goto _exit; - memcpy(*ppOut, pBuf4, nBuf4); - memcpy(*ppOut + nBuf4, pBuf3, nBuf3); - if (nBuf2) { - memcpy(*ppOut + nBuf4 + nBuf3, pBuf2, nBuf2); - } + memcpy(*ppOut, aBuf[3], nBuf3); + memcpy(*ppOut + nBuf3, aBuf[2], nBuf2); if (nBuf1) { - memcpy(*ppOut + nBuf4 + nBuf3 + nBuf2, pBuf1, nBuf1); + memcpy(*ppOut + nBuf3 + nBuf2, aBuf[1], nBuf1); + } + if (nBuf0) { + memcpy(*ppOut + nBuf3 + nBuf2 + nBuf1, aBuf[0], nBuf0); } } From 875566d36194ccb9d09a602810e85c42b3bee50e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 18:06:18 +0800 Subject: [PATCH 71/92] adjust more --- source/dnode/vnode/src/inc/tsdb.h | 2 + .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 93 ++----------------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 57 ++++++------ 3 files changed, 41 insertions(+), 111 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index f31a373254..9e58ac6711 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -152,6 +152,8 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **p int32_t tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest); int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData); int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); +int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[], + int32_t aBufN[]); // SDiskDataHdr int32_t tPutDiskDataHdr(uint8_t *p, void *ph); int32_t tGetDiskDataHdr(uint8_t *p, void *ph); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index f3911350e6..1c5c9979b5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1424,111 +1424,38 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock pBlkInfo->szBlock = 0; pBlkInfo->szKey = 0; - // ================= DATA ==================== - SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, - .fmtVer = 0, - .suid = pBlockData->suid, - .uid = pBlockData->uid, - .nRow = pBlockData->nRow, - .cmprAlg = cmprAlg}; - - // encode ================= - // columns AND SBlockCol - int32_t nBuf1 = 0; - for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - - ASSERT(pColData->flag); - - if (pColData->flag == HAS_NONE) continue; - - SBlockCol blockCol = {.cid = pColData->cid, - .type = pColData->type, - .smaOn = pColData->smaOn, - .flag = pColData->flag, - .szOrigin = pColData->nData}; - - if (pColData->flag != HAS_NULL) { - code = tsdbCmprColData(pColData, cmprAlg, &blockCol, &pWriter->aBuf[0], nBuf1, &pWriter->aBuf[2]); - if (code) goto _err; - - blockCol.offset = nBuf1; - nBuf1 = nBuf1 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); - } - - code = tRealloc(&pWriter->aBuf[1], hdr.szBlkCol + tPutBlockCol(NULL, &blockCol)); - if (code) goto _err; - hdr.szBlkCol += tPutBlockCol(pWriter->aBuf[1] + hdr.szBlkCol, &blockCol); - } - - int32_t nBuf2 = 0; - if (hdr.szBlkCol > 0) { - nBuf2 = hdr.szBlkCol + sizeof(TSCKSUM); - - code = tRealloc(&pWriter->aBuf[1], nBuf2); - if (code) goto _err; - - taosCalcChecksumAppend(0, pWriter->aBuf[1], nBuf2); - } - - // uid + version + tskey - int32_t nBuf3 = 0; - if (pBlockData->uid == 0) { - code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, - &pWriter->aBuf[2], nBuf3, &hdr.szUid, &pWriter->aBuf[3]); - if (code) goto _err; - } - nBuf3 += hdr.szUid; - - code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, - cmprAlg, &pWriter->aBuf[2], nBuf3, &hdr.szVer, &pWriter->aBuf[3]); + int32_t aBufN[4] = {0}; + code = tCmprBlockData(pBlockData, cmprAlg, NULL, NULL, pWriter->aBuf, aBufN); if (code) goto _err; - nBuf3 += hdr.szVer; - - code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, - cmprAlg, &pWriter->aBuf[2], nBuf3, &hdr.szKey, &pWriter->aBuf[3]); - if (code) goto _err; - nBuf3 += hdr.szKey; - - nBuf3 += sizeof(TSCKSUM); - code = tRealloc(&pWriter->aBuf[2], nBuf3); - if (code) goto _err; - - // hdr - int32_t nBuf4 = tPutDiskDataHdr(NULL, &hdr); - code = tRealloc(&pWriter->aBuf[3], nBuf4); - if (code) goto _err; - tPutDiskDataHdr(pWriter->aBuf[3], &hdr); - taosCalcChecksumAppend(taosCalcChecksum(0, pWriter->aBuf[3], nBuf4), pWriter->aBuf[2], nBuf3); // write ================= TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; - pBlkInfo->szKey = nBuf4 + nBuf3; - pBlkInfo->szBlock = nBuf1 + nBuf2 + nBuf3 + nBuf4; + pBlkInfo->szKey = aBufN[3] + aBufN[2]; + pBlkInfo->szBlock = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3]; - int64_t n = taosWriteFile(pFD, pWriter->aBuf[3], nBuf4); + int64_t n = taosWriteFile(pFD, pWriter->aBuf[3], aBufN[3]); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - n = taosWriteFile(pFD, pWriter->aBuf[2], nBuf3); + n = taosWriteFile(pFD, pWriter->aBuf[2], aBufN[2]); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (nBuf2) { - n = taosWriteFile(pFD, pWriter->aBuf[1], nBuf2); + if (aBufN[1]) { + n = taosWriteFile(pFD, pWriter->aBuf[1], aBufN[1]); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } } - if (nBuf1) { - n = taosWriteFile(pFD, pWriter->aBuf[0], nBuf1); + if (aBufN[0]) { + n = taosWriteFile(pFD, pWriter->aBuf[0], aBufN[0]); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 8355149292..d1757a47b0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1500,7 +1500,8 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColD *ppColData = NULL; } -int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[]) { +int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[], + int32_t aBufN[]) { int32_t code = 0; SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, @@ -1512,7 +1513,7 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, // encode ================= // columns AND SBlockCol - int32_t nBuf0 = 0; + aBufN[0] = 0; for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); @@ -1527,11 +1528,11 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, .szOrigin = pColData->nData}; if (pColData->flag != HAS_NULL) { - code = tsdbCmprColData(pColData, cmprAlg, &blockCol, &aBuf[0], nBuf0, &aBuf[2]); + code = tsdbCmprColData(pColData, cmprAlg, &blockCol, &aBuf[0], aBufN[0], &aBuf[2]); if (code) goto _exit; - blockCol.offset = nBuf0; - nBuf0 = nBuf0 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); + blockCol.offset = aBufN[0]; + aBufN[0] = aBufN[0] + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); } code = tRealloc(&aBuf[1], hdr.szBlkCol + tPutBlockCol(NULL, &blockCol)); @@ -1539,59 +1540,59 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, hdr.szBlkCol += tPutBlockCol(aBuf[1] + hdr.szBlkCol, &blockCol); } - int32_t nBuf1 = 0; + aBufN[1] = 0; if (hdr.szBlkCol > 0) { - nBuf1 = hdr.szBlkCol + sizeof(TSCKSUM); + aBufN[1] = hdr.szBlkCol + sizeof(TSCKSUM); - code = tRealloc(&aBuf[1], nBuf1); + code = tRealloc(&aBuf[1], aBufN[1]); if (code) goto _exit; - taosCalcChecksumAppend(0, aBuf[1], nBuf1); + taosCalcChecksumAppend(0, aBuf[1], aBufN[1]); } // uid + version + tskey - int32_t nBuf2 = 0; + aBufN[2] = 0; if (pBlockData->uid == 0) { code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, - &aBuf[2], nBuf2, &hdr.szUid, &aBuf[3]); + &aBuf[2], aBufN[2], &hdr.szUid, &aBuf[3]); if (code) goto _exit; } - nBuf2 += hdr.szUid; + aBufN[2] += hdr.szUid; code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, - cmprAlg, &aBuf[2], nBuf2, &hdr.szVer, &aBuf[3]); + cmprAlg, &aBuf[2], aBufN[2], &hdr.szVer, &aBuf[3]); if (code) goto _exit; - nBuf2 += hdr.szVer; + aBufN[2] += hdr.szVer; code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, - cmprAlg, &aBuf[2], nBuf2, &hdr.szKey, &aBuf[3]); + cmprAlg, &aBuf[2], aBufN[2], &hdr.szKey, &aBuf[3]); if (code) goto _exit; - nBuf2 += hdr.szKey; + aBufN[2] += hdr.szKey; - nBuf2 += sizeof(TSCKSUM); - code = tRealloc(&aBuf[2], nBuf2); + aBufN[2] += sizeof(TSCKSUM); + code = tRealloc(&aBuf[2], aBufN[2]); if (code) goto _exit; // hdr - int32_t nBuf3 = tPutDiskDataHdr(NULL, &hdr); - code = tRealloc(&aBuf[3], nBuf3); + aBufN[3] = tPutDiskDataHdr(NULL, &hdr); + code = tRealloc(&aBuf[3], aBufN[3]); if (code) goto _exit; tPutDiskDataHdr(aBuf[3], &hdr); - taosCalcChecksumAppend(taosCalcChecksum(0, aBuf[3], nBuf3), aBuf[2], nBuf2); + taosCalcChecksumAppend(taosCalcChecksum(0, aBuf[3], aBufN[3]), aBuf[2], aBufN[2]); // aggragate if (ppOut) { - *szOut = nBuf0 + nBuf1 + nBuf2 + nBuf3; + *szOut = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3]; code = tRealloc(ppOut, *szOut); if (code) goto _exit; - memcpy(*ppOut, aBuf[3], nBuf3); - memcpy(*ppOut + nBuf3, aBuf[2], nBuf2); - if (nBuf1) { - memcpy(*ppOut + nBuf3 + nBuf2, aBuf[1], nBuf1); + memcpy(*ppOut, aBuf[3], aBufN[3]); + memcpy(*ppOut + aBufN[3], aBuf[2], aBufN[2]); + if (aBufN[1]) { + memcpy(*ppOut + aBufN[3] + aBufN[2], aBuf[1], aBufN[1]); } - if (nBuf0) { - memcpy(*ppOut + nBuf3 + nBuf2 + nBuf1, aBuf[0], nBuf0); + if (aBufN[0]) { + memcpy(*ppOut + aBufN[3] + aBufN[2] + aBufN[1], aBuf[0], aBufN[0]); } } From 4acc76b7c54af6a1fa8f26ab8ce28ebe9f0834fa Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 18:35:09 +0800 Subject: [PATCH 72/92] adjust more code --- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbUtil.c | 71 +++++++++++++++++++++++++- 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 9e58ac6711..d1f5cfb122 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -154,6 +154,7 @@ int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlo int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[], int32_t aBufN[]); +int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]); // SDiskDataHdr int32_t tPutDiskDataHdr(uint8_t *p, void *ph); int32_t tGetDiskDataHdr(uint8_t *p, void *ph); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index d1757a47b0..6db9d5e6f4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1600,9 +1600,76 @@ _exit: return code; } -int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData) { +int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]) { int32_t code = 0; - // TODO + + tBlockDataClear(pBlockData); + + int32_t n = 0; + SDiskDataHdr hdr = {0}; + + // SDiskDataHdr + n += tGetDiskDataHdr(pIn + n, &hdr); + if (!taosCheckChecksumWhole(pIn, n + hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM))) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } + ASSERT(hdr.delimiter == TSDB_FILE_DLMT); + + pBlockData->suid = hdr.suid; + pBlockData->uid = hdr.uid; + pBlockData->nRow = hdr.nRow; + + // uid + if (hdr.uid == 0) { + ASSERT(hdr.szUid); + code = tsdbDecmprData(pIn + n, hdr.szUid, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aUid, + sizeof(int64_t) * hdr.nRow, &aBuf[0]); + if (code) goto _exit; + } else { + ASSERT(!hdr.szUid); + } + n += hdr.szUid; + + // version + code = tsdbDecmprData(pIn + n, hdr.szVer, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aVersion, + sizeof(int64_t) * hdr.nRow, &aBuf[0]); + if (code) goto _exit; + n += hdr.szVer; + + // TSKEY + code = tsdbDecmprData(pIn + n, hdr.szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr.cmprAlg, (uint8_t **)&pBlockData->aTSKEY, + sizeof(TSKEY) * hdr.nRow, &aBuf[0]); + if (code) goto _exit; + n = n + hdr.szKey + sizeof(TSCKSUM); + + // loop to decode each column data + if (hdr.szBlkCol == 0) goto _exit; + + int32_t nt = 0; + while (nt < hdr.szBlkCol) { + SBlockCol blockCol = {0}; + nt += tGetBlockCol(pIn + n + nt, &blockCol); + ASSERT(nt <= hdr.szBlkCol); + + SColData *pColData; + code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aIdx), &pColData); + if (code) goto _exit; + + tColDataInit(pColData, blockCol.cid, blockCol.type, blockCol.smaOn); + if (blockCol.flag == HAS_NULL) { + for (int32_t iRow = 0; iRow < hdr.nRow; iRow++) { + code = tColDataAppendValue(pColData, &COL_VAL_NULL(blockCol.cid, blockCol.type)); + if (code) goto _exit; + } + } else { + code = tsdbDecmprColData(pIn + n + hdr.szBlkCol + sizeof(TSCKSUM) + blockCol.offset, &blockCol, hdr.cmprAlg, + hdr.nRow, pColData, &aBuf[0]); + if (code) goto _exit; + } + } + +_exit: return code; } From de97c9253a2a1ac4cd3c92b1db986de966210901 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 20:53:19 +0800 Subject: [PATCH 73/92] make it compile --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 149 +++++++++------------ 1 file changed, 64 insertions(+), 85 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index ce970ea870..434fabfe9b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -439,26 +439,6 @@ struct STsdbSnapWriter { SArray* aDelIdxW; }; -#if 0 -static int32_t tsdbSnapWriteAppendData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { - int32_t code = 0; - int32_t iRow = 0; // todo - int32_t nRow = 0; // todo - SBlockData* pBlockData = NULL; // todo - - while (iRow < nRow) { - code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pBlockData, iRow), NULL); - if (code) goto _err; - } - - return code; - -_err: - tsdbError("vgId:%d, tsdb snapshot write append data for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), - pWriter->pTsdb->path, tstrerror(code)); - return code; -} - static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { int32_t code = 0; @@ -470,13 +450,14 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { if (pWriter->pBlockData) { ASSERT(pWriter->iRow < pWriter->pBlockData->nRow); while (pWriter->iRow < pWriter->pBlockData->nRow) { - code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL); + code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL, + 0); // todo if (code) goto _err; if (pWriter->bDataW.nRow >= pWriter->maxRow * 4 / 5) { - pWriter->blockW.last = 0; - code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, - &pWriter->blockW, pWriter->cmprAlg); + // pWriter->blockW.last = 0; + // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, + // &pWriter->blockW, pWriter->cmprAlg); if (code) goto _err; code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); @@ -492,16 +473,16 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { // write remain data if has if (pWriter->bDataW.nRow > 0) { - pWriter->blockW.last = 0; + // pWriter->blockW.last = 0; if (pWriter->bDataW.nRow < pWriter->minRow) { if (pWriter->iBlock > pWriter->mBlock.nItem) { - pWriter->blockW.last = 1; + // pWriter->blockW.last = 1; } } - code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, - &pWriter->blockW, pWriter->cmprAlg); - if (code) goto _err; + // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, + // &pWriter->blockW, pWriter->cmprAlg); + // if (code) goto _err; code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); if (code) goto _err; @@ -513,16 +494,16 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { SBlock block; tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock); - if (block.last) { - code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); - if (code) goto _err; + // if (block.last) { + // code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); + // if (code) goto _err; - tBlockReset(&block); - block.last = 1; - code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pWriter->pBlockIdxW, &block, - pWriter->cmprAlg); - if (code) goto _err; - } + // tBlockReset(&block); + // block.last = 1; + // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pWriter->pBlockIdxW, &block, + // pWriter->cmprAlg); + // if (code) goto _err; + // } code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); if (code) goto _err; @@ -531,8 +512,8 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { } // SBlock - code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, pWriter->pBlockIdxW); - if (code) goto _err; + // code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, pWriter->pBlockIdxW); + // if (code) goto _err; // SBlockIdx if (taosArrayPush(pWriter->aBlockIdxW, pWriter->pBlockIdxW) == NULL) { @@ -562,16 +543,17 @@ static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* p for (int32_t iBlock = 0; iBlock < pWriter->mBlock.nItem; iBlock++) { tMapDataGetItemByIdx(&pWriter->mBlock, iBlock, &block, tGetBlock); - if (block.last) { - code = tsdbReadBlockData(pWriter->pDataFReader, pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); - if (code) goto _err; + // if (block.last) { + // code = tsdbReadBlockData(pWriter->pDataFReader, pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); + // if (code) goto _err; - tBlockReset(&block); - block.last = 1; - code = - tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pBlockIdx, &block, pWriter->cmprAlg); - if (code) goto _err; - } + // tBlockReset(&block); + // block.last = 1; + // code = + // tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pBlockIdx, &block, + // pWriter->cmprAlg); + // if (code) goto _err; + // } code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); if (code) goto _err; @@ -579,7 +561,7 @@ static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* p // SBlock SBlockIdx blockIdx = {.suid = pBlockIdx->suid, .uid = pBlockIdx->uid}; - code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, &blockIdx); + code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, &blockIdx); if (code) goto _err; // SBlockIdx @@ -604,9 +586,9 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { TSDBROW row; TSDBROW* pRow = &row; - // correct schema - code = tBlockDataCorrectSchema(&pWriter->bDataW, pBlockData); - if (code) goto _err; + // // correct schema + // code = tBlockDataCorrectSchema(&pWriter->bDataW, pBlockData); + // if (code) goto _err; // loop to merge *pRow = tsdbRowFromBlockData(pBlockData, iRow); @@ -621,8 +603,8 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { ASSERT(c); if (c < 0) { - code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); - if (code) goto _err; + // code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); + // if (code) goto _err; iRow++; if (iRow < pWriter->pBlockData->nRow) { @@ -631,8 +613,8 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { pRow = NULL; } } else if (c > 0) { - code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL); - if (code) goto _err; + // code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), + // NULL); if (code) goto _err; pWriter->iRow++; if (pWriter->iRow >= pWriter->pBlockData->nRow) { @@ -650,16 +632,15 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock); - if (block.last) { - pWriter->pBlockData = &pWriter->bDataR; + // if (block.last) { + // pWriter->pBlockData = &pWriter->bDataR; - code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, NULL); - if (code) goto _err; - pWriter->iRow = 0; + // code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, + // NULL); if (code) goto _err; pWriter->iRow = 0; - pWriter->iBlock++; - break; - } + // pWriter->iBlock++; + // break; + // } c = tsdbKeyCmprFn(&block.maxKey, &key); @@ -667,10 +648,10 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { if (c < 0) { if (pWriter->bDataW.nRow) { - pWriter->blockW.last = 0; - code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, - &pWriter->blockW, pWriter->cmprAlg); - if (code) goto _err; + // pWriter->blockW.last = 0; + // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, + // &pWriter->blockW, pWriter->cmprAlg); + // if (code) goto _err; code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); if (code) goto _err; @@ -690,9 +671,10 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { if (c > 0) { pWriter->pBlockData = &pWriter->bDataR; - code = - tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, NULL); - if (code) goto _err; + // code = + // tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, + // NULL); + // if (code) goto _err; pWriter->iRow = 0; pWriter->iBlock++; @@ -703,8 +685,8 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { if (pWriter->pBlockData) continue; - code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); - if (code) goto _err; + // code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); + // if (code) goto _err; iRow++; if (iRow < pBlockData->nRow) { @@ -718,9 +700,9 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { if (pWriter->bDataW.nRow < pWriter->maxRow * 4 / 5) continue; _write_block: - code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, - &pWriter->blockW, pWriter->cmprAlg); - if (code) goto _err; + // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, + // &pWriter->blockW, pWriter->cmprAlg); + // if (code) goto _err; code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); if (code) goto _err; @@ -876,8 +858,8 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 // decode SBlockData* pBlockData = &pWriter->bData; - n = tGetBlockData(pData + sizeof(SSnapDataHdr) + sizeof(TABLEID), pBlockData); - ASSERT(n + sizeof(SSnapDataHdr) + sizeof(TABLEID) == nData); + // n = tGetBlockData(pData + sizeof(SSnapDataHdr) + sizeof(TABLEID), pBlockData); + // ASSERT(n + sizeof(SSnapDataHdr) + sizeof(TABLEID) == nData); // open file TSDBKEY keyFirst = tBlockDataFirstKey(pBlockData); @@ -922,16 +904,16 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 if (pSet) { wSet.diskId = pSet->diskId; wSet.fid = fid; - fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0, .loffset = 0}; + fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0, .offset = 0}; fData = *pSet->pDataF; fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0}; fSma = *pSet->pSmaF; } else { wSet.diskId = (SDiskID){.level = 0, .id = 0}; wSet.fid = fid; - fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0, .loffset = 0}; + fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0}; fData = (SDataFile){.commitID = pWriter->commitID, .size = 0}; - fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0}; + fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0, .offset = 0}; fSma = (SSmaFile){.commitID = pWriter->commitID, .size = 0}; } @@ -956,7 +938,6 @@ _err: tstrerror(code)); return code; } -#endif static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; @@ -1206,7 +1187,6 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) int32_t code = 0; SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; -#if 0 // ts data if (pHdr->type == SNAP_DATA_TSDB) { code = tsdbSnapWriteData(pWriter, pData, nData); @@ -1219,7 +1199,6 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) if (code) goto _err; } } -#endif // del data if (pHdr->type == SNAP_DATA_DEL) { From 94b97c4da476577e05b0253b562a80aa2211e378 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 21:42:14 +0800 Subject: [PATCH 74/92] more code --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 61 +++++++++++++++++----- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 434fabfe9b..4724d9681a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -407,6 +407,7 @@ struct STsdbSnapWriter { int8_t cmprAlg; int64_t commitID; + uint8_t* aBuf[5]; // for data file SBlockData bData; @@ -420,6 +421,7 @@ struct STsdbSnapWriter { SBlockData* pBlockData; int32_t iRow; SBlockData bDataR; + SArray* aBlockL; // SArray SDataFWriter* pDataFWriter; SBlockIdx* pBlockIdxW; // NULL when no committing table @@ -429,6 +431,7 @@ struct STsdbSnapWriter { SMapData mBlockW; // SMapData SArray* aBlockIdxW; // SArray + SArray* aBlockLW; // SArray // for del file SDelFReader* pDelFReader; @@ -816,9 +819,11 @@ static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) { if (pWriter->pDataFWriter == NULL) goto _exit; + // finish current table code = tsdbSnapWriteTableDataEnd(pWriter); if (code) goto _err; + // move remain table while (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) { code = tsdbSnapMoveWriteTableData(pWriter, (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx)); if (code) goto _err; @@ -826,8 +831,16 @@ static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) { pWriter->iBlockIdx++; } - code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdxW); - if (code) goto _err; + // write remain stuff + if (taosArrayGetSize(pWriter->aBlockLW) > 0) { + code = tsdbWriteBlockL(pWriter->pDataFWriter, pWriter->aBlockIdxW); + if (code) goto _err; + } + + if (taosArrayGetSize(pWriter->aBlockIdx) > 0) { + code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdxW); + if (code) goto _err; + } code = tsdbFSUpsertFSet(&pWriter->fs, &pWriter->pDataFWriter->wSet); if (code) goto _err; @@ -851,19 +864,22 @@ _err: } static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { - int32_t code = 0; - STsdb* pTsdb = pWriter->pTsdb; - TABLEID id = *(TABLEID*)(pData + sizeof(SSnapDataHdr)); - int64_t n; + int32_t code = 0; + STsdb* pTsdb = pWriter->pTsdb; + SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; + TABLEID id = *(TABLEID*)(pData + sizeof(SSnapDataHdr)); + int64_t n; // decode SBlockData* pBlockData = &pWriter->bData; - // n = tGetBlockData(pData + sizeof(SSnapDataHdr) + sizeof(TABLEID), pBlockData); - // ASSERT(n + sizeof(SSnapDataHdr) + sizeof(TABLEID) == nData); + code = tDecmprBlockData(pData + sizeof(SSnapDataHdr) + sizeof(TABLEID), pHdr->size - sizeof(TABLEID), pBlockData, + pWriter->aBuf); + if (code) goto _err; // open file - TSDBKEY keyFirst = tBlockDataFirstKey(pBlockData); - TSDBKEY keyLast = tBlockDataLastKey(pBlockData); + TSDBKEY keyFirst = {.version = pBlockData->aVersion[0], .ts = pBlockData->aTSKEY[0]}; + TSDBKEY keyLast = {.version = pBlockData->aVersion[pBlockData->nRow - 1], + .ts = pBlockData->aTSKEY[pBlockData->nRow - 1]}; int32_t fid = tsdbKeyFid(keyFirst.ts, pWriter->minutes, pWriter->precision); ASSERT(fid == tsdbKeyFid(keyLast.ts, pWriter->minutes, pWriter->precision)); @@ -882,9 +898,13 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx); if (code) goto _err; + + code = tsdbReadBlockL(pWriter->pDataFReader, pWriter->aBlockL); + if (code) goto _err; } else { ASSERT(pWriter->pDataFReader == NULL); taosArrayClear(pWriter->aBlockIdx); + taosArrayClear(pWriter->aBlockL); } pWriter->iBlockIdx = 0; pWriter->pBlockIdx = NULL; @@ -921,6 +941,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 if (code) goto _err; taosArrayClear(pWriter->aBlockIdxW); + taosArrayClear(pWriter->aBlockLW); tMapDataReset(&pWriter->mBlockW); pWriter->pBlockIdxW = NULL; tBlockDataReset(&pWriter->bDataW); @@ -1110,6 +1131,12 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr code = tBlockDataCreate(&pWriter->bDataR); if (code) goto _err; + pWriter->aBlockL = taosArrayInit(0, sizeof(SBlockL)); + if (pWriter->aBlockL == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pWriter->aBlockIdxW = taosArrayInit(0, sizeof(SBlockIdx)); if (pWriter->aBlockIdxW == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -1118,6 +1145,12 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr code = tBlockDataCreate(&pWriter->bDataW); if (code) goto _err; + pWriter->aBlockLW = taosArrayInit(0, sizeof(SBlockL)); + if (pWriter->aBlockLW == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + // for del file pWriter->aDelIdxR = taosArrayInit(0, sizeof(SDelIdx)); if (pWriter->aDelIdxR == NULL) { @@ -1147,8 +1180,7 @@ _err: } int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { - int32_t code = 0; -#if 0 + int32_t code = 0; STsdbSnapWriter* pWriter = *ppWriter; if (rollback) { @@ -1169,6 +1201,10 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { if (code) goto _err; } + for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t*); iBuf++) { + tFree(pWriter->aBuf[iBuf]); + } + tsdbInfo("vgId:%d, vnode snapshot tsdb writer close for %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path); taosMemoryFree(pWriter); *ppWriter = NULL; @@ -1179,7 +1215,6 @@ _err: pWriter->pTsdb->path, tstrerror(code)); taosMemoryFree(pWriter); *ppWriter = NULL; -#endif return code; } From 831d7dafcad8b128913fc59cec81cfad2e3062ac Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 21:56:37 +0800 Subject: [PATCH 75/92] more code --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 4724d9681a..9a64623062 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -422,6 +422,8 @@ struct STsdbSnapWriter { int32_t iRow; SBlockData bDataR; SArray* aBlockL; // SArray + int32_t iBlockL; + SBlockData lDataR; SDataFWriter* pDataFWriter; SBlockIdx* pBlockIdxW; // NULL when no committing table @@ -912,7 +914,9 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 pWriter->iBlock = 0; pWriter->pBlockData = NULL; pWriter->iRow = 0; + pWriter->iBlockL = 0; tBlockDataReset(&pWriter->bDataR); + tBlockDataReset(&pWriter->lDataR); // write SHeadFile fHead; From 5e4b7bf9bdf01fdeff981e63f6be360924176554 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 22:29:46 +0800 Subject: [PATCH 76/92] more code --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 9a64623062..6a05ad96a6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -79,11 +79,21 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { } else { pReader->pBlockIdx = NULL; } + pReader->iBlockL = 0; - if (pReader->iBlockL < taosArrayGetSize(pReader->aBlockL)) { + while (true) { + if (pReader->iBlockL >= taosArrayGetSize(pReader->aBlockL)) { + pReader->pBlockL = NULL; + break; + } + pReader->pBlockL = (SBlockL*)taosArrayGet(pReader->aBlockL, pReader->iBlockL); - } else { - pReader->pBlockL = NULL; + if (pReader->pBlockL->minVer <= pReader->ever && pReader->pBlockL->maxVer >= pReader->sver) { + // TODO + break; + } + + pReader->iBlockL++; } tsdbInfo("vgId:%d, vnode snapshot tsdb open data file to read for %s, fid:%d", TD_VID(pTsdb->pVnode), pTsdb->path, From 515e8d36b0ad7be6f0471571e747616f94a64d35 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 20 Aug 2022 11:14:00 +0800 Subject: [PATCH 77/92] fix(query): fix bug in merge buffer data and last block. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 273 ++++++++++++++----------- 1 file changed, 154 insertions(+), 119 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index db943645e4..42c74c4a23 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -15,7 +15,10 @@ #include "osDef.h" #include "tsdb.h" + #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) +#define ALL_ROWS_CHECKED_INDEX (INT16_MIN) +#define DEFAULT_ROW_INDEX_VAL (-1) typedef enum { EXTERNAL_ROWS_PREV = 0x1, @@ -220,7 +223,7 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK } for (int32_t j = 0; j < numOfTables; ++j) { - STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid, .indexInBlockL = -1}; + STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid, .indexInBlockL = DEFAULT_ROW_INDEX_VAL}; if (ASCENDING_TRAVERSE(pTsdbReader->order)) { if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReader->window.skey) { info.lastKey = pTsdbReader->window.skey; @@ -699,10 +702,10 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* } { - // 1. time range check, todo add later -// if (pLastBlock->.minKey.ts > pReader->window.ekey || block.maxKey.ts < pReader->window.skey) { -// continue; -// } + // 1. time range check + if (pLastBlock->minKey > pReader->window.ekey || pLastBlock->maxKey < pReader->window.skey) { + continue; + } // 2. version range check if (pLastBlock->minVer > pReader->verRange.maxVer || pLastBlock->maxVer < pReader->verRange.minVer) { @@ -727,12 +730,10 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* return TSDB_CODE_SUCCESS; } -// todo remove pblock parameter -static void setBlockAllDumped(SFileBlockDumpInfo* pDumpInfo, SBlock* pBlock, int32_t order) { +static void setBlockAllDumped(SFileBlockDumpInfo* pDumpInfo, int64_t maxKey, int32_t order) { int32_t step = ASCENDING_TRAVERSE(order) ? 1 : -1; - pDumpInfo->allDumped = true; - pDumpInfo->lastKey = pBlock->maxKey.ts + step; + pDumpInfo->lastKey = maxKey + step; } static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_t colIndex, SColVal* pColVal, @@ -832,7 +833,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn pResBlock->info.rows = remain; pDumpInfo->rowIndex += step * remain; - setBlockAllDumped(pDumpInfo, pBlock, pReader->order); + setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; pReader->cost.blockLoadTime += elapsedTime; @@ -1289,7 +1290,7 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBloc // todo here we need to each key in the last files to identify if it is really overlapped with last block bool overlapWithlastBlock = false; - if (/*hasDataInLastBlock(pLastBlockReader)*/taosArrayGetSize(pLastBlockReader->pBlockL) > 0) { + if (taosArrayGetSize(pLastBlockReader->pBlockL) > 0) { SBlockL *pBlockL = taosArrayGet(pLastBlockReader->pBlockL, pLastBlockReader->currentBlockIndex); overlapWithlastBlock = !(pBlock->maxKey.ts < pBlockL->minKey || pBlock->minKey.ts > pBlockL->maxKey); } @@ -1417,7 +1418,8 @@ static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInf bool init = false; - // file block ---> last block -----> imem -----> mem + // ASC: file block ---> last block -----> imem -----> mem + //DESC: mem -----> imem -----> last block -----> file block if (pReader->order == TSDB_ORDER_ASC) { if (minKey == key) { init = true; @@ -1549,8 +1551,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return TSDB_CODE_SUCCESS; } -// todo handle the desc order check -static int32_t doMergeThreeLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { +static int32_t doMergeMultiLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { SRowMerger merge = {0}; STSRow* pTSRow = NULL; @@ -1569,56 +1570,114 @@ static int32_t doMergeThreeLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo TSDBKEY k = TSDBROW_KEY(pRow); TSDBKEY ik = TSDBROW_KEY(piRow); - int64_t minKey = INT64_MAX; - if (minKey > k.ts) { - minKey = k.ts; + int64_t minKey = 0;//INT64_MAX; + if (ASCENDING_TRAVERSE(pReader->order)) { + minKey = INT64_MAX; // let's find the minimum + if (minKey > k.ts) { + minKey = k.ts; + } + + if (minKey > ik.ts) { + minKey = ik.ts; + } + + if (minKey > key && pBlockData->nRow > 0) { + minKey = key; + } + + if (minKey > tsLast && pLastBlockData->nRow > 0) { + minKey = tsLast; + } + } else { + minKey = INT64_MIN; // let find the maximum ts value + if (minKey < k.ts) { + minKey = k.ts; + } + + if (minKey < ik.ts) { + minKey = ik.ts; + } + + if (minKey < key && pBlockData->nRow > 0) { + minKey = key; + } + + if (minKey < tsLast && pLastBlockData->nRow > 0) { + minKey = tsLast; + } } - if (minKey > ik.ts) { - minKey = ik.ts; - } - - if (minKey > key) { - minKey = key; - } - - if (minKey > tsLast) { - minKey = tsLast; - } - - // file block ---> last block -----> imem -----> mem bool init = false; - if (minKey == key) { - init = true; - TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - tRowMergerInit(&merge, &fRow, pReader->pSchema); - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - } - if (minKey == tsLast) { - if (!init) { + // ASC: file block -----> last block -----> imem -----> mem + // DESC: mem -----> imem -----> last block -----> file block + if (ASCENDING_TRAVERSE(pReader->order)) { + if (minKey == key) { init = true; - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); - tRowMergerInit(&merge, &fRow1, pReader->pSchema); + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } - doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); - } - if (minKey == ik.ts) { - if (!init) { + if (minKey == tsLast) { + if (!init) { + init = true; + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + tRowMergerInit(&merge, &fRow1, pReader->pSchema); + } + doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); + } + + if (minKey == ik.ts) { + if (!init) { + init = true; + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); + tRowMergerInit(&merge, piRow, pSchema); + } + doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); + } + + if (minKey == k.ts) { + if (!init) { + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + tRowMergerInit(&merge, pRow, pSchema); + } + doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + } + } else { + if (minKey == k.ts) { init = true; - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, piRow, pSchema); - } - doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); - } - - if (minKey == k.ts) { - if (!init) { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); tRowMergerInit(&merge, pRow, pSchema); + doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + } + + if (minKey == ik.ts) { + if (!init) { + init = true; + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); + tRowMergerInit(&merge, piRow, pSchema); + } + doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); + } + + if (minKey == tsLast) { + if (!init) { + init = true; + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + tRowMergerInit(&merge, &fRow1, pReader->pSchema); + } + doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); + } + + if (minKey == key) { + if (!init) { + init = true; + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + tRowMergerInit(&merge, &fRow, pReader->pSchema); + } + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } - doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); } tRowMergerGetRow(&merge, &pTSRow); @@ -1816,7 +1875,6 @@ static void initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid } } -#define ALL_ROWS_CHECKED_INDEX INT16_MIN static void setAllRowsChecked(SLastBlockReader *pLastBlockReader) { *pLastBlockReader->rowIndex = ALL_ROWS_CHECKED_INDEX; } @@ -1863,16 +1921,6 @@ static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { return false; } -#if 0 -static int32_t saveCurrentState(SLastBlockReader* pLastBlockReader) { - return pLastBlockReader->rowIndex; -} - -static void restoreState(SLastBlockReader* pLastBlockReader, int32_t state) { - pLastBlockReader->rowIndex = state; -} -#endif - static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { SBlockData* pBlockData = &pLastBlockReader->lastBlockData; return pBlockData->aTSKEY[*pLastBlockReader->rowIndex]; @@ -1894,7 +1942,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI TSDBROW* piRow = getValidRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); if (pBlockScanInfo->iter.hasVal && pBlockScanInfo->iiter.hasVal) { - return doMergeThreeLevelRowsRv(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); + return doMergeMultiLevelRowsRv(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); } else { // imem + file + last block if (pBlockScanInfo->iiter.hasVal) { @@ -2014,7 +2062,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; - initLastBlockReader(pLastBlockReader, pBlockScanInfo->uid, &pBlockScanInfo->indexInBlockL); +// initLastBlockReader(pLastBlockReader, pBlockScanInfo->uid, &pBlockScanInfo->indexInBlockL); // bool has = nextRowInLastBlock(pLastBlockReader); // todo handle if no data in last block SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; @@ -2026,19 +2074,27 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { while (1) { // todo check the validate of row in file block { - if (pBlockData->nRow > 0 && !isValidFileBlockRow(pBlockData, pDumpInfo, pBlockScanInfo, pReader)) { + bool hasBlockData = false; + + while (pBlockData->nRow > 0) { + if (isValidFileBlockRow(pBlockData, pDumpInfo, pBlockScanInfo, pReader)) { + hasBlockData = true; + break; + } + pDumpInfo->rowIndex += step; SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); if (pDumpInfo->rowIndex >= pBlock->nRow || pDumpInfo->rowIndex < 0) { - setBlockAllDumped(pDumpInfo, pBlock, pReader->order); + setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); break; } - - continue; } - if (!hasDataInLastBlock(pLastBlockReader)) { + bool hasBlockLData = hasDataInLastBlock(pLastBlockReader); + + // no data in last block and block, no need to proceed. + if ((hasBlockData == false) && (hasBlockLData == false)) { break; } } @@ -2048,7 +2104,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { // currently loaded file data block is consumed if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); - setBlockAllDumped(pDumpInfo, pBlock, pReader->order); + setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); break; } @@ -2218,11 +2274,11 @@ _err: return code; } -static TSDBKEY getCurrentKeyInBuf(SDataBlockIter* pBlockIter, STsdbReader* pReader) { +static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { TSDBKEY key = {.ts = TSKEY_INITIAL_VAL}; - SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); - STableBlockScanInfo* pScanInfo = taosHashGet(pReader->status.pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); +// SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); +// STableBlockScanInfo* pScanInfo = taosHashGet(pReader->status.pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); initMemDataIterator(pScanInfo, pReader); TSDBROW* pRow = getValidRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader); @@ -2321,16 +2377,17 @@ static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, uint64 code = tBlockDataInit(&pLastBlockReader->lastBlockData, pReader->suid, pReader->suid ? 0 : uid, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { - //todo add log + tsdbError("%p init block data failed, code:%s %s", pReader, tstrerror(code), pReader->idStr); return code; } code = tsdbReadLastBlock(pReader->pFileReader, pBlock, &pLastBlockReader->lastBlockData); if (code != TSDB_CODE_SUCCESS) { - // tsdbDebug("%p error occurs in loading last block into buffer, last block index:%d, total:%d brange:%" PRId64 "-%" PRId64 - // ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", code:%s %s", - // pReader, *index, pBlockIter->numOfBlocks.numOfLastBlocks, 0, 0, pBlock->nRow, - // pBlock->minVer, pBlock->maxVer, tstrerror(code), pReader->idStr); + tsdbError( + "%p error occurs in loading last block into buffer, last block index:%d, total:%d rows:%d, minVer:%" PRId64 + ", maxVer:%" PRId64 ", code:%s %s", + pReader, pLastBlockReader->currentBlockIndex, (int32_t)taosArrayGetSize(pBlocks), pBlock->nRow, pBlock->minVer, + pBlock->maxVer, tstrerror(code), pReader->idStr); } return TSDB_CODE_SUCCESS; @@ -2357,7 +2414,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { } initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); - if (pScanInfo->indexInBlockL == -1 || pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) { + if (pScanInfo->indexInBlockL == DEFAULT_ROW_INDEX_VAL || pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) { bool hasData = nextRowInLastBlock(pLastBlockReader); if (!hasData) { // current table does not have rows in last block, try next table pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); @@ -2398,9 +2455,17 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; if (pBlockInfo != NULL) { - pScanInfo = taosHashGet(pStatus->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + pScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + } else { + pScanInfo = pReader->status.pTableIter; + } + + if (pBlockInfo != NULL) { pBlock = getCurrentBlock(pBlockIter); - key = getCurrentKeyInBuf(pBlockIter, pReader); + } + + { + key = getCurrentKeyInBuf(pScanInfo, pReader); // load the last data block of current table code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo->uid, pReader); @@ -2408,7 +2473,10 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { return code; } - initLastBlockReader(pLastBlockReader, pBlockInfo->uid, &pScanInfo->indexInBlockL); + initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); + if (pScanInfo->indexInBlockL == DEFAULT_ROW_INDEX_VAL || pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) { + bool hasData = nextRowInLastBlock(pLastBlockReader); + } } if (pBlockInfo == NULL) { // build data block from last data file @@ -2439,7 +2507,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { pInfo->uid = pScanInfo->uid; pInfo->window = (STimeWindow){.skey = pBlock->minKey.ts, .ekey = pBlock->maxKey.ts}; setComposedBlockFlag(pReader, false); - setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlock, pReader->order); + setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlock->maxKey.ts, pReader->order); } return code; @@ -2663,39 +2731,6 @@ SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_ return (SVersionRange){.minVer = startVer, .maxVer = endVer}; } -// // todo not unref yet, since it is not support multi-group interpolation query -// static UNUSED_FUNC void changeQueryHandleForInterpQuery(STsdbReader* pHandle) { -// // filter the queried time stamp in the first place -// STsdbReader* pTsdbReadHandle = (STsdbReader*)pHandle; - -// // starts from the buffer in case of descending timestamp order check data blocks -// size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); - -// int32_t i = 0; -// while (i < numOfTables) { -// STableBlockScanInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); - -// // the first qualified table for interpolation query -// // if ((pTsdbReadHandle->window.skey <= pCheckInfo->pTableObj->lastKey) && -// // (pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL)) { -// // break; -// // } - -// i++; -// } - -// // there are no data in all the tables -// if (i == numOfTables) { -// return; -// } - -// STableBlockScanInfo info = *(STableBlockScanInfo*)taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); -// taosArrayClear(pTsdbReadHandle->pTableCheckInfo); - -// info.lastKey = pTsdbReadHandle->window.skey; -// taosArrayPush(pTsdbReadHandle->pTableCheckInfo, &info); -// } - bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order) { ASSERT(pKey != NULL); if (pDelList == NULL) { @@ -3227,7 +3262,7 @@ int32_t tsdbSetTableId(STsdbReader* pReader, int64_t uid) { ASSERT(pReader != NULL); taosHashClear(pReader->status.pTableMap); - STableBlockScanInfo info = {.lastKey = 0, .uid = uid}; + STableBlockScanInfo info = {.lastKey = 0, .uid = uid, .indexInBlockL = DEFAULT_ROW_INDEX_VAL}; taosHashPut(pReader->status.pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info)); return TDB_CODE_SUCCESS; } From 09ffcd2e13c23e9a0a3a67d39496140e9eb92d07 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 20 Aug 2022 12:41:58 +0800 Subject: [PATCH 78/92] fix(query): fix bug in tsdbread --- source/dnode/vnode/src/tsdb/tsdbRead.c | 66 ++++++++++++++++---------- tests/script/tsim/parser/function.sim | 1 + 2 files changed, 43 insertions(+), 24 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 42c74c4a23..4a6f831ac4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1428,16 +1428,20 @@ static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInf } if (minKey == tsLast) { - if (!init) { + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + if (init) { + tRowMerge(&merge, &fRow1); + } else { init = true; - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); tRowMergerInit(&merge, &fRow1, pReader->pSchema); } doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); } if (minKey == k.ts) { - if (!init) { + if (init) { + tRowMerge(&merge, pRow); + } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); tRowMergerInit(&merge, pRow, pSchema); @@ -1453,16 +1457,20 @@ static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInf } if (minKey == tsLast) { - if (!init) { + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + if (init) { + tRowMerge(&merge, &fRow1); + } else { init = true; - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); tRowMergerInit(&merge, &fRow1, pReader->pSchema); } doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); } if (minKey == key) { - if (!init) { + if (init) { + tRowMerge(&merge, &fRow); + } else { init = true; tRowMergerInit(&merge, &fRow, pReader->pSchema); } @@ -1620,16 +1628,20 @@ static int32_t doMergeMultiLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo } if (minKey == tsLast) { - if (!init) { + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + if (init) { + tRowMerge(&merge, &fRow1); + } else { init = true; - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); tRowMergerInit(&merge, &fRow1, pReader->pSchema); } doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); } if (minKey == ik.ts) { - if (!init) { + if (init) { + tRowMerge(&merge, piRow); + } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); tRowMergerInit(&merge, piRow, pSchema); @@ -1638,7 +1650,9 @@ static int32_t doMergeMultiLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo } if (minKey == k.ts) { - if (!init) { + if (init) { + tRowMerge(&merge, pRow); + } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); tRowMergerInit(&merge, pRow, pSchema); } @@ -1653,7 +1667,9 @@ static int32_t doMergeMultiLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo } if (minKey == ik.ts) { - if (!init) { + if (init) { + tRowMerge(&merge, piRow); + } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); tRowMergerInit(&merge, piRow, pSchema); @@ -1662,19 +1678,22 @@ static int32_t doMergeMultiLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo } if (minKey == tsLast) { - if (!init) { + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + if (init) { + tRowMerge(&merge, &fRow1); + } else { init = true; - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); tRowMergerInit(&merge, &fRow1, pReader->pSchema); } doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); } if (minKey == key) { + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); if (!init) { - init = true; - TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); tRowMergerInit(&merge, &fRow, pReader->pSchema); + } else { + tRowMerge(&merge, &fRow); } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } @@ -1885,11 +1904,12 @@ static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { return false; } + bool asc = ASCENDING_TRAVERSE(pLastBlockReader->order); *(pLastBlockReader->rowIndex) += step; SBlockData* pBlockData = &pLastBlockReader->lastBlockData; for(int32_t i = *(pLastBlockReader->rowIndex); i < pBlockData->nRow && i >= 0; i += step) { - if (pBlockData->aUid[i] != pLastBlockReader->uid) { + if (pBlockData->aUid != NULL && pBlockData->aUid[i] != pLastBlockReader->uid) { continue; } @@ -1901,15 +1921,14 @@ static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { continue; } - // no data any more + // no data any more, todo opt handle desc case if (pBlockData->aTSKEY[i] > pLastBlockReader->window.ekey) { - setAllRowsChecked(pLastBlockReader); - return false; + continue; } + // todo opt handle desc case if (pBlockData->aVersion[i] > pLastBlockReader->verRange.maxVer) { - setAllRowsChecked(pLastBlockReader); - return false; + continue; } *(pLastBlockReader->rowIndex) = i; @@ -2076,7 +2095,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { { bool hasBlockData = false; - while (pBlockData->nRow > 0) { + while (pBlockData->nRow > 0) { // find the first qualified row in data block if (isValidFileBlockRow(pBlockData, pDumpInfo, pBlockScanInfo, pReader)) { hasBlockData = true; break; @@ -2119,8 +2138,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { setComposedBlockFlag(pReader, true); int64_t et = taosGetTimestampUs(); - tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 - " rows:%d, elapsed time:%.2f ms %s", + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%d, elapsed time:%.2f ms %s", pReader, pBlockScanInfo->uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, (et - st) / 1000.0, pReader->idStr); diff --git a/tests/script/tsim/parser/function.sim b/tests/script/tsim/parser/function.sim index 0219a84c64..a27810edeb 100644 --- a/tests/script/tsim/parser/function.sim +++ b/tests/script/tsim/parser/function.sim @@ -70,6 +70,7 @@ if $data00 != @15-08-18 00:00:00.000@ then return -1 endi if $data01 != 2.068333156 then + print expect 2.068333156, actual: $data01 return -1 endi if $data02 != 2.063999891 then From 0dfd03206863cd77f9de40d114e34af7a068391b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 20 Aug 2022 16:57:44 +0800 Subject: [PATCH 79/92] fix(query): reset block data when no blocks are available. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 43 +++++++++++++++++--------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 4a6f831ac4..0a11185eff 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -400,7 +400,7 @@ _err: static void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, SHashObj* pTableMap) { pIter->order = order; pIter->index = -1; - pIter->numOfBlocks = -1; + pIter->numOfBlocks = 0; if (pIter->blockList == NULL) { pIter->blockList = taosArrayInit(4, sizeof(SFileDataBlockInfo)); } else { @@ -1904,7 +1904,6 @@ static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { return false; } - bool asc = ASCENDING_TRAVERSE(pLastBlockReader->order); *(pLastBlockReader->rowIndex) += step; SBlockData* pBlockData = &pLastBlockReader->lastBlockData; @@ -2320,10 +2319,12 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); SArray* pIndexList = taosArrayInit(numOfTables, sizeof(SBlockIdx)); + SArray* pLastBlocks = pStatus->fileIter.pLastBlockReader->pBlockL; while (1) { bool hasNext = filesetIteratorNext(&pStatus->fileIter, pReader); if (!hasNext) { // no data files on disk + taosArrayClear(pLastBlocks); break; } @@ -2334,7 +2335,6 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { return code; } - SArray* pLastBlocks = pStatus->fileIter.pLastBlockReader->pBlockL; code = tsdbReadBlockL(pReader->pFileReader, pLastBlocks); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pIndexList); @@ -2385,6 +2385,7 @@ static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, uint64 } if (pLastBlockReader->currentBlockIndex == -1) { + tBlockDataDestroy(&pLastBlockReader->lastBlockData, false); return TSDB_CODE_SUCCESS; } @@ -2431,16 +2432,25 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { return code; } - initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); - if (pScanInfo->indexInBlockL == DEFAULT_ROW_INDEX_VAL || pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) { - bool hasData = nextRowInLastBlock(pLastBlockReader); - if (!hasData) { // current table does not have rows in last block, try next table - pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); - if (pStatus->pTableIter == NULL) { - return TSDB_CODE_SUCCESS; + if (pLastBlockReader->currentBlockIndex != -1) { + initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); + if (pScanInfo->indexInBlockL == DEFAULT_ROW_INDEX_VAL || + pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) { + bool hasData = nextRowInLastBlock(pLastBlockReader); + if (!hasData) { // current table does not have rows in last block, try next table + pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); + if (pStatus->pTableIter == NULL) { + return TSDB_CODE_SUCCESS; + } + continue; } - continue; } + } else { // no data in last block, try next table + pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); + if (pStatus->pTableIter == NULL) { + return TSDB_CODE_SUCCESS; + } + continue; } code = doBuildDataBlock(pReader); @@ -2491,6 +2501,11 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { return code; } + if (pLastBlockReader->currentBlockIndex == -1) { +// ASSERT(0); + printf("error\n"); + } + initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); if (pScanInfo->indexInBlockL == DEFAULT_ROW_INDEX_VAL || pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) { bool hasData = nextRowInLastBlock(pLastBlockReader); @@ -2594,7 +2609,8 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl if (num.numOfBlocks > 0) { code = initBlockIterator(pReader, pBlockIter, num.numOfBlocks); } else { - pBlockIter->numOfBlocks = 0; + tBlockDataReset(&pReader->status.fileBlockData); + resetDataBlockIterator(pBlockIter, pReader->order, pReader->status.pTableMap); } // set the correct start position according to the query time window @@ -2663,9 +2679,8 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { initBlockDumpInfo(pReader, pBlockIter); } else if (taosArrayGetSize(pReader->status.fileIter.pLastBlockReader->pBlockL) > 0) { // data blocks in current file are exhausted, let's try the next file now // todo dump all data in last block if exists. - pBlockIter->numOfBlocks = 0; - taosArrayClear(pBlockIter->blockList); tBlockDataReset(&pReader->status.fileBlockData); + resetDataBlockIterator(pBlockIter, pReader->order, pReader->status.pTableMap); goto _begin; } else { code = initForFirstBlockInFile(pReader, pBlockIter); From df0d6f18234f0e95ac446d400d4dd9181bc0464e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 20 Aug 2022 19:05:55 +0800 Subject: [PATCH 80/92] fix(query): fix memory leak. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 0a11185eff..62e7defb5b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -338,6 +338,11 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb pLReader->order = pReader->order; pLReader->window = pReader->window; pLReader->verRange = pReader->verRange; + + int32_t code = tBlockDataCreate(&pLReader->lastBlockData); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, pReader->idStr); @@ -593,7 +598,7 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader, size_t num = taosArrayGetSize(aBlockIdx); if (num == 0) { - taosArrayClear(aBlockIdx); + taosArrayDestroy(aBlockIdx); return TSDB_CODE_SUCCESS; } @@ -2359,6 +2364,8 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { taosArrayDestroy(pQLastBlock); break; } + + taosArrayDestroy(pQLastBlock); } // no blocks in current file, try next files @@ -2385,16 +2392,11 @@ static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, uint64 } if (pLastBlockReader->currentBlockIndex == -1) { - tBlockDataDestroy(&pLastBlockReader->lastBlockData, false); + tBlockDataClear(&pLastBlockReader->lastBlockData); return TSDB_CODE_SUCCESS; } - int32_t code = tBlockDataCreate(&pLastBlockReader->lastBlockData); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = tBlockDataInit(&pLastBlockReader->lastBlockData, pReader->suid, pReader->suid ? 0 : uid, pReader->pSchema); + int32_t code = tBlockDataInit(&pLastBlockReader->lastBlockData, pReader->suid, pReader->suid ? 0 : uid, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { tsdbError("%p init block data failed, code:%s %s", pReader, tstrerror(code), pReader->idStr); return code; @@ -3463,6 +3465,13 @@ void tsdbReaderClose(STsdbReader* pReader) { tsdbDataFReaderClose(&pReader->pFileReader); } + SFilesetIter* pFilesetIter = &pReader->status.fileIter; + if (pFilesetIter->pLastBlockReader != NULL) { + tBlockDataDestroy(&pFilesetIter->pLastBlockReader->lastBlockData, true); + taosArrayDestroy(pFilesetIter->pLastBlockReader->pBlockL); + taosMemoryFree(pFilesetIter->pLastBlockReader); + } + SIOCostSummary* pCost = &pReader->cost; tsdbDebug("%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 From 03879752a16e5f46811bed88c3d7cd3a9fa3b26a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 20 Aug 2022 19:17:17 +0800 Subject: [PATCH 81/92] fix(query): fix syntax error on the darwin platform. --- source/dnode/vnode/src/tsdb/tsdbCache.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 4e4a88b4a5..ed25783e9f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -266,14 +266,14 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb } for (++iCol; iCol < nCol; ++iCol) { - SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); - if (keyTs >= tTsVal->ts) { - SColVal *tColVal = &tTsVal->colVal; + SLastCol *tTsVal1 = (SLastCol *)taosArrayGet(pLast, iCol); + if (keyTs >= tTsVal1->ts) { + SColVal *tColVal = &tTsVal1->colVal; SColVal colVal = {0}; tTSRowGetVal(row, pTSchema, iCol, &colVal); if (colVal.isNone || colVal.isNull) { - if (keyTs == tTsVal->ts && !tColVal->isNone && !tColVal->isNull) { + if (keyTs == tTsVal1->ts && !tColVal->isNone && !tColVal->isNull) { invalidate = true; break; @@ -958,7 +958,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->idx = (SBlockIdx){.suid = suid, .uid = uid}; - pIter->fsLastState.state = SFSNEXTROW_FS; + pIter->fsLastState.state = (SFSLASTNEXTROWSTATES) SFSNEXTROW_FS; pIter->fsLastState.pTsdb = pTsdb; pIter->fsLastState.aDFileSet = pIter->pReadSnap->fs.aDFileSet; pIter->fsLastState.pBlockIdxExp = &pIter->idx; @@ -1063,9 +1063,9 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow) { int iMerge[4] = {-1, -1, -1, -1}; int nMerge = 0; for (int i = 0; i < nMax; ++i) { - TSDBKEY maxKey = TSDBROW_KEY(max[i]); + TSDBKEY maxKey1 = TSDBROW_KEY(max[i]); - bool deleted = tsdbKeyDeleted(&maxKey, pIter->pSkyline, &pIter->iSkyline); + bool deleted = tsdbKeyDeleted(&maxKey1, pIter->pSkyline, &pIter->iSkyline); if (!deleted) { iMerge[nMerge] = iMax[i]; merge[nMerge++] = max[i]; From 6d8a70ebf645025f57b0a7b1929dc6f979a1f0a4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 20 Aug 2022 20:22:39 +0800 Subject: [PATCH 82/92] fix(query): remove invalid free. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 62e7defb5b..765bc68d19 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2392,7 +2392,7 @@ static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, uint64 } if (pLastBlockReader->currentBlockIndex == -1) { - tBlockDataClear(&pLastBlockReader->lastBlockData); + tBlockDataReset(&pLastBlockReader->lastBlockData); return TSDB_CODE_SUCCESS; } From c3ec89c3cbfee316589642fc56eeaf9300222008 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 20 Aug 2022 21:18:12 +0800 Subject: [PATCH 83/92] fix a coredump --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 1c5c9979b5..a8969b2d9c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -802,7 +802,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo if (pBlockCol == NULL || pBlockCol->cid > pColData->cid) { // add a lot of NONE for (int32_t iRow = 0; iRow < hdr.nRow; iRow++) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pBlockCol->cid, pBlockCol->type)); + code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); if (code) goto _err; } } else { From 16d28b8a396a90ba608a72afcc68ab6834a62bf7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 20 Aug 2022 22:24:07 +0800 Subject: [PATCH 84/92] fix: a bug --- source/dnode/vnode/src/meta/metaQuery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 9bbff6c6bd..805bc24d8c 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -548,7 +548,7 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv int32_t nKey = 0; tdbTbcGet(pSkmDbC, &pKey, &nKey, NULL, NULL); - if (((SSkmDbKey *)pKey)->uid != uid) { + if (((SSkmDbKey *)pKey)->uid != skmDbKey.uid) { metaULock(pMeta); tdbTbcClose(pSkmDbC); code = TSDB_CODE_NOT_FOUND; From 5a1ba3d0163eccd4b1772943a46f8ab03713d150 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 20 Aug 2022 23:26:58 +0800 Subject: [PATCH 85/92] fix(query): increase the output buffer size. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 765bc68d19..c7e6d381a2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -463,7 +463,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->pTsdb = getTsdbByRetentions(pVnode, pCond->twindows.skey, pVnode->config.tsdbCfg.retentions, idstr, &level); pReader->suid = pCond->suid; pReader->order = pCond->order; - pReader->capacity = 1; + pReader->capacity = 4096; pReader->idStr = (idstr != NULL) ? strdup(idstr) : NULL; pReader->verRange = getQueryVerRange(pVnode, pCond, level); pReader->type = pCond->type; From 38b30d071ed52c4360d2721c70f49c7bfe5c81ce Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 21 Aug 2022 00:02:43 +0800 Subject: [PATCH 86/92] fix(query): fix errors. --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 2 +- source/libs/function/src/builtinsimpl.c | 1 + tests/script/tsim/parser/function.sim | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 6a05ad96a6..ab2b2b617a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -938,7 +938,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 if (pSet) { wSet.diskId = pSet->diskId; wSet.fid = fid; - fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0, .offset = 0}; + fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0}; fData = *pSet->pDataF; fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0}; fSma = *pSet->pSmaF; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index bf4a07f8e2..5051dcd65c 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -5574,6 +5574,7 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { if (pCtx->end.key != INT64_MIN) { pInfo->dOutput += twa_get_area(pInfo->p, pCtx->end); pInfo->p = pCtx->end; + numOfElems += 1; } pInfo->win.ekey = pInfo->p.key; diff --git a/tests/script/tsim/parser/function.sim b/tests/script/tsim/parser/function.sim index a27810edeb..0002a5d095 100644 --- a/tests/script/tsim/parser/function.sim +++ b/tests/script/tsim/parser/function.sim @@ -129,6 +129,7 @@ if $data03 != 2 then return -1 endi if $data11 != 2.077099980 then + print expect 2.077099980, actual: $data11 return -1 endi if $data12 != 2.077000022 then From 5324889c52abe2833f657063194c8c84f696ce8b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 21 Aug 2022 11:26:20 +0800 Subject: [PATCH 87/92] fix: sma file corrupt --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index a8969b2d9c..c8f3862071 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -694,8 +694,18 @@ int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnD code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; + // seek + int64_t n = taosLSeekFile(pReader->pSmaFD, pSmaInfo->offset, SEEK_SET); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } else if (n < pSmaInfo->offset) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + // read - int64_t n = taosReadFile(pReader->pSmaFD, pReader->aBuf[0], size); + n = taosReadFile(pReader->pSmaFD, pReader->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; From 470c774f54fb03a842c8752ea32f334a51e7bd91 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 21 Aug 2022 15:05:56 +0800 Subject: [PATCH 88/92] fix: alter table schema --- source/dnode/vnode/src/meta/metaTable.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index c1e59adbb0..aa107ab253 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -357,10 +357,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { metaSaveToTbDb(pMeta, &nStbEntry); // update uid index - SMetaInfo info; - metaGetEntryInfo(&nStbEntry, &info); - tdbTbcUpsert(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), - &(SUidIdxVal){.suid = info.suid, .version = info.version, .skmVer = info.skmVer}, sizeof(SUidIdxVal), 0); + metaUpdateUidIdx(pMeta, &nStbEntry); if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); metaULock(pMeta); @@ -884,7 +881,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA } SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid}; - tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, ((STag*)(ctbEntry.ctbEntry.pTags))->len, &pMeta->txn); + tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, + ((STag *)(ctbEntry.ctbEntry.pTags))->len, &pMeta->txn); tDecoderClear(&dc1); tDecoderClear(&dc2); @@ -1091,7 +1089,8 @@ static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid}; - return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags, ((STag*)(pME->ctbEntry.pTags))->len, &pMeta->txn); + return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags, + ((STag *)(pME->ctbEntry.pTags))->len, &pMeta->txn); } int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, From fe918b3f69443f2af549c404c22c5b0d5b0356e3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 21 Aug 2022 15:56:06 +0800 Subject: [PATCH 89/92] fix(query): set correct column data. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c7e6d381a2..0e2a4e47bf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -814,19 +814,20 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn pColData = taosArrayGet(pResBlock->pDataBlock, i); SColData* pData = tBlockDataGetColDataByIdx(pBlockData, colIndex); - - if (pData->cid == pColData->info.colId) { + if (pData->cid < pColData->info.colId) { + colIndex += 1; + } else if (pData->cid == pColData->info.colId) { for (int32_t j = pDumpInfo->rowIndex; j < endIndex && j >= 0; j += step) { tColDataGetValue(pData, j, &cv); doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo); } colIndex += 1; + i += 1; ASSERT(rowIndex == remain); } else { // the specified column does not exist in file block, fill with null data colDataAppendNNULL(pColData, 0, remain); + i += 1; } - - i += 1; } while (i < numOfOutputCols) { @@ -2436,8 +2437,8 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { if (pLastBlockReader->currentBlockIndex != -1) { initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); - if (pScanInfo->indexInBlockL == DEFAULT_ROW_INDEX_VAL || - pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) { + int32_t index = pScanInfo->indexInBlockL; + if (index == DEFAULT_ROW_INDEX_VAL || index == pLastBlockReader->lastBlockData.nRow) { bool hasData = nextRowInLastBlock(pLastBlockReader); if (!hasData) { // current table does not have rows in last block, try next table pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); From d305e975c44a7982a2d70e92b88dea04a3d44914 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 21 Aug 2022 15:56:46 +0800 Subject: [PATCH 90/92] fix(query): remove printf. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 0e2a4e47bf..d89b45b140 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2506,7 +2506,6 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { if (pLastBlockReader->currentBlockIndex == -1) { // ASSERT(0); - printf("error\n"); } initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); From 7904686915e34967973e87e2d6e09c63d7cb6427 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 21 Aug 2022 18:59:21 +0800 Subject: [PATCH 91/92] fix(query): support delete in last --- source/dnode/vnode/src/tsdb/tsdbRead.c | 54 +++++++++++++------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index d89b45b140..6c9042f64f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -166,7 +166,7 @@ static int buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, i static TSDBROW* getValidRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader); static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader, SRowMerger* pMerger); -static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, int64_t ts, SRowMerger* pMerger); +static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, SRowMerger* pMerger); static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, STsdbReader* pReader); static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid); @@ -1441,7 +1441,7 @@ static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInf init = true; tRowMergerInit(&merge, &fRow1, pReader->pSchema); } - doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge); } if (minKey == k.ts) { @@ -1470,7 +1470,7 @@ static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInf init = true; tRowMergerInit(&merge, &fRow1, pReader->pSchema); } - doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge); } if (minKey == key) { @@ -1641,7 +1641,7 @@ static int32_t doMergeMultiLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo init = true; tRowMergerInit(&merge, &fRow1, pReader->pSchema); } - doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge); } if (minKey == ik.ts) { @@ -1691,7 +1691,7 @@ static int32_t doMergeMultiLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo init = true; tRowMergerInit(&merge, &fRow1, pReader->pSchema); } - doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge); } if (minKey == key) { @@ -1904,7 +1904,7 @@ static void setAllRowsChecked(SLastBlockReader *pLastBlockReader) { *pLastBlockReader->rowIndex = ALL_ROWS_CHECKED_INDEX; } -static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { +static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) { int32_t step = (pLastBlockReader->order == TSDB_ORDER_ASC) ? 1 : -1; if (*pLastBlockReader->rowIndex == ALL_ROWS_CHECKED_INDEX) { return false; @@ -1918,21 +1918,28 @@ static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { continue; } - if (pBlockData->aTSKEY[i] < pLastBlockReader->window.skey) { + int64_t ts = pBlockData->aTSKEY[i]; + if (ts < pLastBlockReader->window.skey) { continue; } - if (pBlockData->aVersion[i] < pLastBlockReader->verRange.minVer) { + int64_t ver = pBlockData->aVersion[i]; + if (ver < pLastBlockReader->verRange.minVer) { continue; } // no data any more, todo opt handle desc case - if (pBlockData->aTSKEY[i] > pLastBlockReader->window.ekey) { + if (ts > pLastBlockReader->window.ekey) { continue; } // todo opt handle desc case - if (pBlockData->aVersion[i] > pLastBlockReader->verRange.maxVer) { + if (ver > pLastBlockReader->verRange.maxVer) { + continue; + } + + TSDBKEY k = {.ts = ts, .version = ver}; + if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, &k, pLastBlockReader->order)) { continue; } @@ -1958,7 +1965,8 @@ static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { } // todo refactor -static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader *pLastBlockReader) { +static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, + SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; int64_t key = (pBlockData->nRow > 0)? pBlockData->aTSKEY[pDumpInfo->rowIndex]:INT64_MIN; @@ -2011,7 +2019,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); tRowMergerInit(&merge, &fRow1, pReader->pSchema); - doMergeRowsInLastBlock(pLastBlockReader, ts, &merge); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge); tRowMergerGetRow(&merge, &pTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); @@ -2025,7 +2033,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI tRowMergerInit(&merge, &fRow, pReader->pSchema); doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - doMergeRowsInLastBlock(pLastBlockReader, ts, &merge); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge); tRowMergerGetRow(&merge, &pTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); @@ -2061,7 +2069,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI TSDBROW fRow = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); tRowMergerInit(&merge, &fRow, pReader->pSchema); - doMergeRowsInLastBlock(pLastBlockReader, tsLastBlock, &merge); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge); tRowMergerGetRow(&merge, &pTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); @@ -2085,10 +2093,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { pBlockScanInfo = pReader->status.pTableIter; } - SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; -// initLastBlockReader(pLastBlockReader, pBlockScanInfo->uid, &pBlockScanInfo->indexInBlockL); -// bool has = nextRowInLastBlock(pLastBlockReader); // todo handle if no data in last block - + SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBlockData* pBlockData = &pReader->status.fileBlockData; int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; @@ -2439,7 +2444,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); int32_t index = pScanInfo->indexInBlockL; if (index == DEFAULT_ROW_INDEX_VAL || index == pLastBlockReader->lastBlockData.nRow) { - bool hasData = nextRowInLastBlock(pLastBlockReader); + bool hasData = nextRowInLastBlock(pLastBlockReader, pScanInfo); if (!hasData) { // current table does not have rows in last block, try next table pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); if (pStatus->pTableIter == NULL) { @@ -2504,13 +2509,10 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { return code; } - if (pLastBlockReader->currentBlockIndex == -1) { -// ASSERT(0); - } - + // note: the lastblock may be null here initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); if (pScanInfo->indexInBlockL == DEFAULT_ROW_INDEX_VAL || pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) { - bool hasData = nextRowInLastBlock(pLastBlockReader); + bool hasData = nextRowInLastBlock(pLastBlockReader, pScanInfo); } } @@ -3034,8 +3036,8 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc } // todo check if the rows are dropped or not -int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, int64_t ts, SRowMerger* pMerger) { - while(nextRowInLastBlock(pLastBlockReader)) { +int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, SRowMerger* pMerger) { + while(nextRowInLastBlock(pLastBlockReader, pScanInfo)) { int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 == ts) { TSDBROW fRow1 = tsdbRowFromBlockData(&pLastBlockReader->lastBlockData, *pLastBlockReader->rowIndex); From 29a1245be4d2ac830481e0c5d61b73fb5e00bdf2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 21 Aug 2022 20:15:53 +0800 Subject: [PATCH 92/92] fix(query): use the separate last block delete index. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 6c9042f64f..fc7595627d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -45,7 +45,8 @@ typedef struct STableBlockScanInfo { SIterInfo iter; // mem buffer skip list iterator SIterInfo iiter; // imem buffer skip list iterator SArray* delSkyline; // delete info for this table - int32_t fileDelIndex; + int32_t fileDelIndex; // file block delete index + int32_t lastBlockDelIndex;// delete index for last block bool iterInit; // whether to initialize the in-memory skip list iterator or not int16_t indexInBlockL;// row position in last block } STableBlockScanInfo; @@ -1296,7 +1297,7 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBloc // todo here we need to each key in the last files to identify if it is really overlapped with last block bool overlapWithlastBlock = false; - if (taosArrayGetSize(pLastBlockReader->pBlockL) > 0) { + if (taosArrayGetSize(pLastBlockReader->pBlockL) > 0 && (pLastBlockReader->currentBlockIndex != -1)) { SBlockL *pBlockL = taosArrayGet(pLastBlockReader->pBlockL, pLastBlockReader->currentBlockIndex); overlapWithlastBlock = !(pBlock->maxKey.ts < pBlockL->minKey || pBlock->minKey.ts > pBlockL->maxKey); } @@ -1939,7 +1940,7 @@ static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader, STableBlockSc } TSDBKEY k = {.ts = ts, .version = ver}; - if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, &k, pLastBlockReader->order)) { + if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order)) { continue; } @@ -2295,6 +2296,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* ASCENDING_TRAVERSE(pReader->order) ? 0 : taosArrayGetSize(pBlockScanInfo->delSkyline) - 1; pBlockScanInfo->iiter.index = pBlockScanInfo->iter.index; pBlockScanInfo->fileDelIndex = pBlockScanInfo->iter.index; + pBlockScanInfo->lastBlockDelIndex = pBlockScanInfo->iter.index; return code; _err: @@ -2305,9 +2307,6 @@ _err: static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { TSDBKEY key = {.ts = TSKEY_INITIAL_VAL}; -// SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); -// STableBlockScanInfo* pScanInfo = taosHashGet(pReader->status.pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); - initMemDataIterator(pScanInfo, pReader); TSDBROW* pRow = getValidRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader); if (pRow != NULL) { @@ -2381,10 +2380,12 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { return TSDB_CODE_SUCCESS; } -static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, uint64_t uid, STsdbReader* pReader) { +static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo *pBlockScanInfo, STsdbReader* pReader) { SArray* pBlocks = pLastBlockReader->pBlockL; SBlockL* pBlock = NULL; + uint64_t uid = pBlockScanInfo->uid; + initMemDataIterator(pBlockScanInfo, pReader); pLastBlockReader->currentBlockIndex = -1; // find the correct SBlockL @@ -2435,7 +2436,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { // load the last data block of current table // todo opt perf by avoiding load last block repeatly STableBlockScanInfo* pScanInfo = pStatus->pTableIter; - int32_t code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo->uid, pReader); + int32_t code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2504,7 +2505,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { key = getCurrentKeyInBuf(pScanInfo, pReader); // load the last data block of current table - code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo->uid, pReader); + code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo, pReader); if (code != TSDB_CODE_SUCCESS) { return code; }