From 3b0008a6252282adc925ebaa23ca1e52c1a8a30d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 4 Sep 2022 18:42:23 +0800 Subject: [PATCH] more code --- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 30 ++++++++------ source/dnode/vnode/src/tsdb/tsdbUtil.c | 40 +++---------------- 2 files changed, 23 insertions(+), 47 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 1a9d540af4..1433d7c0be 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -934,18 +934,20 @@ _err: return code; } -static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo, int8_t fromLast, - SBlockData *pBlockData) { +static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo, SBlockData *pBlockData) { int32_t code = 0; tBlockDataClear(pBlockData); - STsdbFD *pFD = fromLast ? pReader->aSstFD[0] : pReader->pDataFD; // (todo) - - // todo: realloc pReader->aBuf[0] + STsdbFD *pFD = pReader->pDataFD; // uid + version + tskey - tsdbReadFile(pFD, pBlkInfo->offset, pReader->aBuf[0], pBlkInfo->szKey); // todo + code = tRealloc(&pReader->aBuf[0], pBlkInfo->szKey); + if (code) goto _err; + + code = tsdbReadFile(pFD, pBlkInfo->offset, pReader->aBuf[0], pBlkInfo->szKey); + if (code) goto _err; + SDiskDataHdr hdr; uint8_t *p = pReader->aBuf[0] + tGetDiskDataHdr(pReader->aBuf[0], &hdr); @@ -978,14 +980,15 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo if (code) goto _err; p += hdr.szKey; - ASSERT(p - pReader->aBuf[0] == pBlkInfo->szKey - sizeof(TSCKSUM)); + ASSERT(p - pReader->aBuf[0] == pBlkInfo->szKey); // read and decode columns if (taosArrayGetSize(pBlockData->aIdx) == 0) goto _exit; if (hdr.szBlkCol > 0) { int64_t offset = pBlkInfo->offset + pBlkInfo->szKey; - tsdbReadFile(pFD, offset, pReader->aBuf[0], hdr.szBlkCol + sizeof(TSCKSUM)); + code = tsdbReadFile(pFD, offset, pReader->aBuf[0], hdr.szBlkCol); + if (code) goto _err; } SBlockCol blockCol = {.cid = 0}; @@ -1022,10 +1025,11 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo } } else { // decode from binary - int64_t offset = pBlkInfo->offset + pBlkInfo->szKey + hdr.szBlkCol + sizeof(TSCKSUM) + pBlockCol->offset; - int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM); + int64_t offset = pBlkInfo->offset + pBlkInfo->szKey + hdr.szBlkCol + pBlockCol->offset; + int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue; - tsdbReadFile(pFD, offset, pReader->aBuf[1], size); + code = tsdbReadFile(pFD, offset, pReader->aBuf[1], size); + if (code) goto _err; code = tsdbDecmprColData(pReader->aBuf[1], pBlockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->aBuf[2]); if (code) goto _err; @@ -1044,7 +1048,7 @@ _err: int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData) { int32_t code = 0; - code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[0], 0, pBlockData); + code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[0], pBlockData); if (code) goto _err; if (pDataBlk->nSubBlock > 1) { @@ -1062,7 +1066,7 @@ int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData tBlockDataInitEx(&bData2, pBlockData); for (int32_t iSubBlock = 1; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) { - code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[iSubBlock], 0, &bData1); + code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[iSubBlock], &bData1); if (code) { tBlockDataDestroy(&bData1, 1); tBlockDataDestroy(&bData2, 1); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 9b3094bb2c..8509c0c759 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1548,7 +1548,7 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, if (code) goto _exit; blockCol.offset = aBufN[0]; - aBufN[0] = aBufN[0] + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); + aBufN[0] = aBufN[0] + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue; } code = tRealloc(&aBuf[1], hdr.szBlkCol + tPutBlockCol(NULL, &blockCol)); @@ -1556,15 +1556,8 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, hdr.szBlkCol += tPutBlockCol(aBuf[1] + hdr.szBlkCol, &blockCol); } - aBufN[1] = 0; - if (hdr.szBlkCol > 0) { - aBufN[1] = hdr.szBlkCol + sizeof(TSCKSUM); - - code = tRealloc(&aBuf[1], aBufN[1]); - if (code) goto _exit; - - taosCalcChecksumAppend(0, aBuf[1], aBufN[1]); - } + // SBlockCol + aBufN[1] = hdr.szBlkCol; // uid + version + tskey aBufN[2] = 0; @@ -1585,16 +1578,11 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, if (code) goto _exit; aBufN[2] += hdr.szKey; - aBufN[2] += sizeof(TSCKSUM); - code = tRealloc(&aBuf[2], aBufN[2]); - if (code) goto _exit; - // hdr aBufN[3] = tPutDiskDataHdr(NULL, &hdr); code = tRealloc(&aBuf[3], aBufN[3]); if (code) goto _exit; tPutDiskDataHdr(aBuf[3], &hdr); - taosCalcChecksumAppend(taosCalcChecksum(0, aBuf[3], aBufN[3]), aBuf[2], aBufN[2]); // aggragate if (ppOut) { @@ -1626,10 +1614,6 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin // SDiskDataHdr n += tGetDiskDataHdr(pIn + n, &hdr); - if (!taosCheckChecksumWhole(pIn, n + hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM))) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; - } ASSERT(hdr.delimiter == TSDB_FILE_DLMT); pBlockData->suid = hdr.suid; @@ -1657,7 +1641,7 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin code = tsdbDecmprData(pIn + n, hdr.szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr.cmprAlg, (uint8_t **)&pBlockData->aTSKEY, sizeof(TSKEY) * hdr.nRow, &aBuf[0]); if (code) goto _exit; - n = n + hdr.szKey + sizeof(TSCKSUM); + n += hdr.szKey; // loop to decode each column data if (hdr.szBlkCol == 0) goto _exit; @@ -1679,8 +1663,8 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin if (code) goto _exit; } } else { - code = tsdbDecmprColData(pIn + n + hdr.szBlkCol + sizeof(TSCKSUM) + blockCol.offset, &blockCol, hdr.cmprAlg, - hdr.nRow, pColData, &aBuf[0]); + code = tsdbDecmprColData(pIn + n + hdr.szBlkCol + blockCol.offset, &blockCol, hdr.cmprAlg, hdr.nRow, pColData, + &aBuf[0]); if (code) goto _exit; } } @@ -2062,12 +2046,6 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol } size += pBlockCol->szValue; - // checksum - size += sizeof(TSCKSUM); - code = tRealloc(ppOut, nOut + size); - if (code) goto _exit; - taosCalcChecksumAppend(0, *ppOut + nOut, size); - _exit: return code; } @@ -2076,12 +2054,6 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in uint8_t **ppBuf) { int32_t code = 0; - int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM); - if (!taosCheckChecksumWhole(pIn, size)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; - } - ASSERT(pColData->cid == pBlockCol->cid); ASSERT(pColData->type == pBlockCol->type); pColData->smaOn = pBlockCol->smaOn;