From 5b1492cb4b6fdc626b5f805cf7356be8f96634f3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 25 Jun 2023 15:09:15 +0800 Subject: [PATCH] refact code --- source/dnode/vnode/src/tsdb/tsdbDataFileRW.c | 172 ++++++------- source/dnode/vnode/src/tsdb/tsdbSttFileRW.c | 252 +++++++++++-------- source/dnode/vnode/src/tsdb/tsdbUpgrade.c | 13 +- 3 files changed, 230 insertions(+), 207 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index a1c1de3d73..47d46ed6e2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -25,6 +25,11 @@ typedef struct { SFDataPtr rsrvd[2]; } STombFooter; +extern int32_t tsdbFileDoWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, + TTombBlkArray *tombBlkArray, uint8_t **bufArr); +extern int32_t tsdbFileDoWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, + int64_t *fileSize); + // SDataFileReader ============================================= struct SDataFileReader { SDataFileReaderConfig config[1]; @@ -644,81 +649,89 @@ _exit: return code; } -static int32_t tsdbDataFileWriteBrinBlock(SDataFileWriter *writer) { - if (BRIN_BLOCK_SIZE(writer->brinBlock) == 0) return 0; +int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAlg, int64_t *fileSize, + TBrinBlkArray *brinBlkArray, uint8_t **bufArr) { + if (BRIN_BLOCK_SIZE(brinBlock) == 0) return 0; - int32_t code = 0; - int32_t lino = 0; + int32_t code; // get SBrinBlk SBrinBlk brinBlk[1] = { { .dp[0] = { - .offset = writer->files[TSDB_FTYPE_HEAD].size, + .offset = *fileSize, .size = 0, }, .minTbid = { - .suid = TARRAY2_FIRST(writer->brinBlock->suid), - .uid = TARRAY2_FIRST(writer->brinBlock->uid), + .suid = TARRAY2_FIRST(brinBlock->suid), + .uid = TARRAY2_FIRST(brinBlock->uid), }, .maxTbid = { - .suid = TARRAY2_LAST(writer->brinBlock->suid), - .uid = TARRAY2_LAST(writer->brinBlock->uid), + .suid = TARRAY2_LAST(brinBlock->suid), + .uid = TARRAY2_LAST(brinBlock->uid), }, - .minVer = TARRAY2_FIRST(writer->brinBlock->minVer), - .maxVer = TARRAY2_FIRST(writer->brinBlock->minVer), - .numRec = BRIN_BLOCK_SIZE(writer->brinBlock), - .cmprAlg = writer->config->cmprAlg, + .minVer = TARRAY2_FIRST(brinBlock->minVer), + .maxVer = TARRAY2_FIRST(brinBlock->minVer), + .numRec = BRIN_BLOCK_SIZE(brinBlock), + .cmprAlg = cmprAlg, }, }; - for (int32_t i = 1; i < BRIN_BLOCK_SIZE(writer->brinBlock); i++) { - if (brinBlk->minVer > TARRAY2_GET(writer->brinBlock->minVer, i)) { - brinBlk->minVer = TARRAY2_GET(writer->brinBlock->minVer, i); + for (int32_t i = 1; i < BRIN_BLOCK_SIZE(brinBlock); i++) { + if (brinBlk->minVer > TARRAY2_GET(brinBlock->minVer, i)) { + brinBlk->minVer = TARRAY2_GET(brinBlock->minVer, i); } - if (brinBlk->maxVer < TARRAY2_GET(writer->brinBlock->maxVer, i)) { - brinBlk->maxVer = TARRAY2_GET(writer->brinBlock->maxVer, i); + if (brinBlk->maxVer < TARRAY2_GET(brinBlock->maxVer, i)) { + brinBlk->maxVer = TARRAY2_GET(brinBlock->maxVer, i); } } // write to file - for (int32_t i = 0; i < ARRAY_SIZE(writer->brinBlock->dataArr1); i++) { - code = tsdbCmprData((uint8_t *)TARRAY2_DATA(writer->brinBlock->dataArr1 + i), - TARRAY2_DATA_LEN(writer->brinBlock->dataArr1 + i), TSDB_DATA_TYPE_BIGINT, brinBlk->cmprAlg, - &writer->config->bufArr[0], 0, &brinBlk->size[i], &writer->config->bufArr[1]); - TSDB_CHECK_CODE(code, lino, _exit); + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr1); i++) { + code = tsdbCmprData((uint8_t *)TARRAY2_DATA(brinBlock->dataArr1 + i), TARRAY2_DATA_LEN(brinBlock->dataArr1 + i), + TSDB_DATA_TYPE_BIGINT, brinBlk->cmprAlg, &bufArr[0], 0, &brinBlk->size[i], &bufArr[1]); + if (code) return code; - code = tsdbWriteFile(writer->fd[TSDB_FTYPE_HEAD], writer->files[TSDB_FTYPE_HEAD].size, writer->config->bufArr[0], - brinBlk->size[i]); - TSDB_CHECK_CODE(code, lino, _exit); + code = tsdbWriteFile(fd, *fileSize, bufArr[0], brinBlk->size[i]); + if (code) return code; brinBlk->dp->size += brinBlk->size[i]; - writer->files[TSDB_FTYPE_HEAD].size += brinBlk->size[i]; + *fileSize += brinBlk->size[i]; } - for (int32_t i = 0, j = ARRAY_SIZE(writer->brinBlock->dataArr1); i < ARRAY_SIZE(writer->brinBlock->dataArr2); - i++, j++) { - code = tsdbCmprData((uint8_t *)TARRAY2_DATA(writer->brinBlock->dataArr2 + i), - TARRAY2_DATA_LEN(writer->brinBlock->dataArr2 + i), TSDB_DATA_TYPE_INT, brinBlk->cmprAlg, - &writer->config->bufArr[0], 0, &brinBlk->size[j], &writer->config->bufArr[1]); - TSDB_CHECK_CODE(code, lino, _exit); + for (int32_t i = 0, j = ARRAY_SIZE(brinBlock->dataArr1); i < ARRAY_SIZE(brinBlock->dataArr2); i++, j++) { + code = tsdbCmprData((uint8_t *)TARRAY2_DATA(brinBlock->dataArr2 + i), TARRAY2_DATA_LEN(brinBlock->dataArr2 + i), + TSDB_DATA_TYPE_INT, brinBlk->cmprAlg, &bufArr[0], 0, &brinBlk->size[j], &bufArr[1]); + if (code) return code; - code = tsdbWriteFile(writer->fd[TSDB_FTYPE_HEAD], writer->files[TSDB_FTYPE_HEAD].size, writer->config->bufArr[0], - brinBlk->size[j]); - TSDB_CHECK_CODE(code, lino, _exit); + code = tsdbWriteFile(fd, *fileSize, bufArr[0], brinBlk->size[j]); + if (code) return code; brinBlk->dp->size += brinBlk->size[j]; - writer->files[TSDB_FTYPE_HEAD].size += brinBlk->size[j]; + *fileSize += brinBlk->size[j]; } // append to brinBlkArray - code = TARRAY2_APPEND_PTR(writer->brinBlkArray, brinBlk); - TSDB_CHECK_CODE(code, lino, _exit); + code = TARRAY2_APPEND_PTR(brinBlkArray, brinBlk); + if (code) return code; - tBrinBlockClear(writer->brinBlock); + tBrinBlockClear(brinBlock); + + return 0; +} + +static int32_t tsdbDataFileWriteBrinBlock(SDataFileWriter *writer) { + if (BRIN_BLOCK_SIZE(writer->brinBlock) == 0) return 0; + + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFileWriteBrinBlock(writer->fd[TSDB_FTYPE_HEAD], writer->brinBlock, writer->config->cmprAlg, + &writer->files[TSDB_FTYPE_HEAD].size, writer->brinBlkArray, writer->config->bufArr); + TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { @@ -1154,53 +1167,10 @@ static int32_t tsdbDataFileDoWriteTombBlock(SDataFileWriter *writer) { int32_t code = 0; int32_t lino = 0; - STombBlk tombBlk[1] = {{ - .numRec = TOMB_BLOCK_SIZE(writer->tombBlock), - .minTbid = - { - .suid = TARRAY2_FIRST(writer->tombBlock->suid), - .uid = TARRAY2_FIRST(writer->tombBlock->uid), - }, - .maxTbid = - { - .suid = TARRAY2_LAST(writer->tombBlock->suid), - .uid = TARRAY2_LAST(writer->tombBlock->uid), - }, - .minVer = TARRAY2_FIRST(writer->tombBlock->version), - .maxVer = TARRAY2_FIRST(writer->tombBlock->version), - .dp[0] = - { - .offset = writer->files[TSDB_FTYPE_TOMB].size, - .size = 0, - }, - }}; - - for (int32_t i = 1; i < TOMB_BLOCK_SIZE(writer->tombBlock); i++) { - tombBlk->minVer = TMIN(tombBlk->minVer, TARRAY2_GET(writer->tombBlock->version, i)); - tombBlk->maxVer = TMAX(tombBlk->maxVer, TARRAY2_GET(writer->tombBlock->version, i)); - } - - for (int32_t i = 0; i < ARRAY_SIZE(writer->tombBlock->dataArr); i++) { - int32_t size; - code = tsdbCmprData((uint8_t *)TARRAY2_DATA(&writer->tombBlock->dataArr[i]), - TARRAY2_DATA_LEN(&writer->tombBlock->dataArr[i]), TSDB_DATA_TYPE_BIGINT, TWO_STAGE_COMP, - &writer->config->bufArr[0], 0, &size, &writer->config->bufArr[1]); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbWriteFile(writer->fd[TSDB_FTYPE_TOMB], writer->files[TSDB_FTYPE_TOMB].size, writer->config->bufArr[0], - size); - TSDB_CHECK_CODE(code, lino, _exit); - - tombBlk->size[i] = size; - tombBlk->dp[0].size += size; - writer->files[TSDB_FTYPE_TOMB].size += size; - } - - code = TARRAY2_APPEND_PTR(writer->tombBlkArray, tombBlk); + code = tsdbFileDoWriteTombBlock(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlock, writer->config->cmprAlg, + &writer->files[TSDB_FTYPE_TOMB].size, writer->tombBlkArray, writer->config->bufArr); TSDB_CHECK_CODE(code, lino, _exit); - tTombBlockClear(writer->tombBlock); - _exit: if (code) { TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); @@ -1214,14 +1184,9 @@ static int32_t tsdbDataFileDoWriteTombBlk(SDataFileWriter *writer) { int32_t code = 0; int32_t lino = 0; - int32_t ftype = TSDB_FTYPE_TOMB; - writer->tombFooter->tombBlkPtr->offset = writer->files[ftype].size; - writer->tombFooter->tombBlkPtr->size = TARRAY2_DATA_LEN(writer->tombBlkArray); - - code = tsdbWriteFile(writer->fd[ftype], writer->tombFooter->tombBlkPtr->offset, - (const uint8_t *)TARRAY2_DATA(writer->tombBlkArray), writer->tombFooter->tombBlkPtr->size); + code = tsdbFileDoWriteTombBlk(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlkArray, writer->tombFooter->tombBlkPtr, + &writer->files[TSDB_FTYPE_TOMB].size); TSDB_CHECK_CODE(code, lino, _exit); - writer->files[ftype].size += writer->tombFooter->tombBlkPtr->size; _exit: if (code) { @@ -1306,20 +1271,25 @@ _exit: return code; } -static int32_t tsdbDataFileWriteBrinBlk(SDataFileWriter *writer) { - ASSERT(TARRAY2_SIZE(writer->brinBlkArray) > 0); +int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize) { + ASSERT(TARRAY2_SIZE(brinBlkArray) > 0); + ptr->offset = *fileSize; + ptr->size = TARRAY2_DATA_LEN(brinBlkArray); + int32_t code = tsdbWriteFile(fd, ptr->offset, (uint8_t *)TARRAY2_DATA(brinBlkArray), ptr->size); + if (code) return code; + + *fileSize += ptr->size; + return 0; +} + +static int32_t tsdbDataFileWriteBrinBlk(SDataFileWriter *writer) { int32_t code = 0; int32_t lino = 0; - int32_t ftype = TSDB_FTYPE_HEAD; - writer->headFooter->brinBlkPtr->offset = writer->files[ftype].size; - writer->headFooter->brinBlkPtr->size = TARRAY2_DATA_LEN(writer->brinBlkArray); - - code = tsdbWriteFile(writer->fd[ftype], writer->headFooter->brinBlkPtr->offset, - (uint8_t *)TARRAY2_DATA(writer->brinBlkArray), writer->headFooter->brinBlkPtr->size); + code = tsdbFileWriteBrinBlk(writer->fd[TSDB_FTYPE_HEAD], writer->brinBlkArray, writer->headFooter->brinBlkPtr, + &writer->files[TSDB_FTYPE_HEAD].size); TSDB_CHECK_CODE(code, lino, _exit); - writer->files[ftype].size += writer->headFooter->brinBlkPtr->size; _exit: if (code) { diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c index 97a2a8b478..326550bac2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c @@ -402,51 +402,64 @@ struct SSttFileWriter { uint8_t *bufArr[5]; }; +int32_t tsdbFileDoWriteBlockData(STsdbFD *fd, SBlockData *blockData, int8_t cmprAlg, int64_t *fileSize, + TSttBlkArray *sttBlkArray, uint8_t **bufArr) { + if (blockData->nRow == 0) return 0; + + int32_t code = 0; + + SSttBlk sttBlk[1] = {{ + .suid = blockData->suid, + .minUid = blockData->uid ? blockData->uid : blockData->aUid[0], + .maxUid = blockData->uid ? blockData->uid : blockData->aUid[blockData->nRow - 1], + .minKey = blockData->aTSKEY[0], + .maxKey = blockData->aTSKEY[0], + .minVer = blockData->aVersion[0], + .maxVer = blockData->aVersion[0], + .nRow = blockData->nRow, + }}; + + for (int32_t iRow = 1; iRow < blockData->nRow; iRow++) { + if (sttBlk->minKey > blockData->aTSKEY[iRow]) sttBlk->minKey = blockData->aTSKEY[iRow]; + if (sttBlk->maxKey < blockData->aTSKEY[iRow]) sttBlk->maxKey = blockData->aTSKEY[iRow]; + if (sttBlk->minVer > blockData->aVersion[iRow]) sttBlk->minVer = blockData->aVersion[iRow]; + if (sttBlk->maxVer < blockData->aVersion[iRow]) sttBlk->maxVer = blockData->aVersion[iRow]; + } + + int32_t sizeArr[5] = {0}; + code = tCmprBlockData(blockData, cmprAlg, NULL, NULL, bufArr, sizeArr); + if (code) return code; + + sttBlk->bInfo.offset = *fileSize; + sttBlk->bInfo.szKey = sizeArr[2] + sizeArr[3]; + sttBlk->bInfo.szBlock = sizeArr[0] + sizeArr[1] + sttBlk->bInfo.szKey; + + for (int32_t i = 3; i >= 0; i--) { + if (sizeArr[i]) { + code = tsdbWriteFile(fd, *fileSize, bufArr[i], sizeArr[i]); + if (code) return code; + *fileSize += sizeArr[i]; + } + } + + code = TARRAY2_APPEND_PTR(sttBlkArray, sttBlk); + if (code) return code; + + tBlockDataClear(blockData); + + return 0; +} + static int32_t tsdbSttFileDoWriteBlockData(SSttFileWriter *writer) { if (writer->blockData->nRow == 0) return 0; int32_t code = 0; int32_t lino = 0; - SSttBlk sttBlk[1] = {{ - .suid = writer->blockData->suid, - .minUid = writer->blockData->uid ? writer->blockData->uid : writer->blockData->aUid[0], - .maxUid = writer->blockData->uid ? writer->blockData->uid : writer->blockData->aUid[writer->blockData->nRow - 1], - .minKey = writer->blockData->aTSKEY[0], - .maxKey = writer->blockData->aTSKEY[0], - .minVer = writer->blockData->aVersion[0], - .maxVer = writer->blockData->aVersion[0], - .nRow = writer->blockData->nRow, - }}; - - for (int32_t iRow = 1; iRow < writer->blockData->nRow; iRow++) { - if (sttBlk->minKey > writer->blockData->aTSKEY[iRow]) sttBlk->minKey = writer->blockData->aTSKEY[iRow]; - if (sttBlk->maxKey < writer->blockData->aTSKEY[iRow]) sttBlk->maxKey = writer->blockData->aTSKEY[iRow]; - if (sttBlk->minVer > writer->blockData->aVersion[iRow]) sttBlk->minVer = writer->blockData->aVersion[iRow]; - if (sttBlk->maxVer < writer->blockData->aVersion[iRow]) sttBlk->maxVer = writer->blockData->aVersion[iRow]; - } - - int32_t sizeArr[5] = {0}; - code = tCmprBlockData(writer->blockData, writer->config->cmprAlg, NULL, NULL, writer->config->bufArr, sizeArr); + code = tsdbFileDoWriteBlockData(writer->fd, writer->blockData, writer->config->cmprAlg, &writer->file->size, + writer->sttBlkArray, writer->config->bufArr); TSDB_CHECK_CODE(code, lino, _exit); - sttBlk->bInfo.offset = writer->file->size; - sttBlk->bInfo.szKey = sizeArr[2] + sizeArr[3]; - sttBlk->bInfo.szBlock = sizeArr[0] + sizeArr[1] + sttBlk->bInfo.szKey; - - for (int32_t i = 3; i >= 0; i--) { - if (sizeArr[i]) { - code = tsdbWriteFile(writer->fd, writer->file->size, writer->config->bufArr[i], sizeArr[i]); - TSDB_CHECK_CODE(code, lino, _exit); - writer->file->size += sizeArr[i]; - } - } - - code = TARRAY2_APPEND_PTR(writer->sttBlkArray, sttBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - tBlockDataClear(writer->blockData); - _exit: if (code) { TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); @@ -516,61 +529,72 @@ _exit: return code; } +int32_t tsdbFileDoWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, + TTombBlkArray *tombBlkArray, uint8_t **bufArr) { + int32_t code; + + if (TOMB_BLOCK_SIZE(tombBlock) == 0) return 0; + + STombBlk tombBlk[1] = {{ + .dp[0] = + { + .offset = *fileSize, + .size = 0, + }, + .minTbid = + { + .suid = TARRAY2_FIRST(tombBlock->suid), + .uid = TARRAY2_FIRST(tombBlock->uid), + }, + .maxTbid = + { + .suid = TARRAY2_LAST(tombBlock->suid), + .uid = TARRAY2_LAST(tombBlock->uid), + }, + .minVer = TARRAY2_FIRST(tombBlock->version), + .maxVer = TARRAY2_FIRST(tombBlock->version), + .numRec = TOMB_BLOCK_SIZE(tombBlock), + .cmprAlg = cmprAlg, + }}; + + for (int32_t i = 1; i < TOMB_BLOCK_SIZE(tombBlock); i++) { + if (tombBlk->minVer > TARRAY2_GET(tombBlock->version, i)) { + tombBlk->minVer = TARRAY2_GET(tombBlock->version, i); + } + if (tombBlk->maxVer < TARRAY2_GET(tombBlock->version, i)) { + tombBlk->maxVer = TARRAY2_GET(tombBlock->version, i); + } + } + + for (int32_t i = 0; i < ARRAY_SIZE(tombBlock->dataArr); i++) { + code = tsdbCmprData((uint8_t *)TARRAY2_DATA(&tombBlock->dataArr[i]), TARRAY2_DATA_LEN(&tombBlock->dataArr[i]), + TSDB_DATA_TYPE_BIGINT, tombBlk->cmprAlg, &bufArr[0], 0, &tombBlk->size[i], &bufArr[1]); + if (code) return code; + + code = tsdbWriteFile(fd, *fileSize, bufArr[0], tombBlk->size[i]); + if (code) return code; + + tombBlk->dp->size += tombBlk->size[i]; + *fileSize += tombBlk->size[i]; + } + + code = TARRAY2_APPEND_PTR(tombBlkArray, tombBlk); + if (code) return code; + + tTombBlockClear(tombBlock); + return 0; +} + static int32_t tsdbSttFileDoWriteTombBlock(SSttFileWriter *writer) { if (TOMB_BLOCK_SIZE(writer->tombBlock) == 0) return 0; int32_t code = 0; int32_t lino = 0; - STombBlk tombBlk[1] = {{ - .dp[0] = - { - .offset = writer->file->size, - .size = 0, - }, - .minTbid = - { - .suid = TARRAY2_FIRST(writer->tombBlock->suid), - .uid = TARRAY2_FIRST(writer->tombBlock->uid), - }, - .maxTbid = - { - .suid = TARRAY2_LAST(writer->tombBlock->suid), - .uid = TARRAY2_LAST(writer->tombBlock->uid), - }, - .minVer = TARRAY2_FIRST(writer->tombBlock->version), - .maxVer = TARRAY2_FIRST(writer->tombBlock->version), - .numRec = TOMB_BLOCK_SIZE(writer->tombBlock), - .cmprAlg = writer->config->cmprAlg, - }}; - - for (int32_t i = 1; i < TOMB_BLOCK_SIZE(writer->tombBlock); i++) { - if (tombBlk->minVer > TARRAY2_GET(writer->tombBlock->version, i)) { - tombBlk->minVer = TARRAY2_GET(writer->tombBlock->version, i); - } - if (tombBlk->maxVer < TARRAY2_GET(writer->tombBlock->version, i)) { - tombBlk->maxVer = TARRAY2_GET(writer->tombBlock->version, i); - } - } - - for (int32_t i = 0; i < ARRAY_SIZE(writer->tombBlock->dataArr); i++) { - code = tsdbCmprData((uint8_t *)TARRAY2_DATA(&writer->tombBlock->dataArr[i]), - TARRAY2_DATA_LEN(&writer->tombBlock->dataArr[i]), TSDB_DATA_TYPE_BIGINT, tombBlk->cmprAlg, - &writer->config->bufArr[0], 0, &tombBlk->size[i], &writer->config->bufArr[1]); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbWriteFile(writer->fd, writer->file->size, writer->config->bufArr[0], tombBlk->size[i]); - TSDB_CHECK_CODE(code, lino, _exit); - - tombBlk->dp->size += tombBlk->size[i]; - writer->file->size += tombBlk->size[i]; - } - - code = TARRAY2_APPEND_PTR(writer->tombBlkArray, tombBlk); + code = tsdbFileDoWriteTombBlock(writer->fd, writer->tombBlock, writer->config->cmprAlg, &writer->file->size, + writer->tombBlkArray, writer->config->bufArr); TSDB_CHECK_CODE(code, lino, _exit); - tTombBlockClear(writer->tombBlock); - _exit: if (code) { TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); @@ -578,18 +602,27 @@ _exit: return code; } +int32_t tsdbFileDoWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize) { + ptr->size = TARRAY2_DATA_LEN(sttBlkArray); + if (ptr->size > 0) { + ptr->offset = *fileSize; + + int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(sttBlkArray), ptr->size); + if (code) { + return code; + } + + *fileSize += ptr->size; + } + return 0; +} + static int32_t tsdbSttFileDoWriteSttBlk(SSttFileWriter *writer) { int32_t code = 0; int32_t lino; - writer->footer->sttBlkPtr->size = TARRAY2_DATA_LEN(writer->sttBlkArray); - if (writer->footer->sttBlkPtr->size) { - writer->footer->sttBlkPtr->offset = writer->file->size; - code = tsdbWriteFile(writer->fd, writer->file->size, (const uint8_t *)TARRAY2_DATA(writer->sttBlkArray), - writer->footer->sttBlkPtr->size); - TSDB_CHECK_CODE(code, lino, _exit); - writer->file->size += writer->footer->sttBlkPtr->size; - } + code = tsdbFileDoWriteSttBlk(writer->fd, writer->sttBlkArray, writer->footer->sttBlkPtr, &writer->file->size); + TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { @@ -618,18 +651,27 @@ _exit: return code; } +int32_t tsdbFileDoWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize) { + ptr->size = TARRAY2_DATA_LEN(tombBlkArray); + if (ptr->size > 0) { + ptr->offset = *fileSize; + + int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(tombBlkArray), ptr->size); + if (code) { + return code; + } + + *fileSize += ptr->size; + } + return 0; +} + static int32_t tsdbSttFileDoWriteTombBlk(SSttFileWriter *writer) { int32_t code = 0; int32_t lino = 0; - writer->footer->tombBlkPtr->size = TARRAY2_DATA_LEN(writer->tombBlkArray); - if (writer->footer->tombBlkPtr->size) { - writer->footer->tombBlkPtr->offset = writer->file->size; - code = tsdbWriteFile(writer->fd, writer->file->size, (const uint8_t *)TARRAY2_DATA(writer->tombBlkArray), - writer->footer->tombBlkPtr->size); - TSDB_CHECK_CODE(code, lino, _exit); - writer->file->size += writer->footer->tombBlkPtr->size; - } + code = tsdbFileDoWriteTombBlk(writer->fd, writer->tombBlkArray, writer->footer->tombBlkPtr, &writer->file->size); + TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { @@ -638,13 +680,17 @@ _exit: return code; } -static int32_t tsdbSttFileDoWriteFooter(SSttFileWriter *writer) { - int32_t code = tsdbWriteFile(writer->fd, writer->file->size, (const uint8_t *)writer->footer, sizeof(writer->footer)); +int32_t tsdbSttFileDoWriteFooterImpl(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize) { + int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer)); if (code) return code; - writer->file->size += sizeof(writer->footer); + *fileSize += sizeof(*footer); return 0; } +static int32_t tsdbSttFileDoWriteFooter(SSttFileWriter *writer) { + return tsdbSttFileDoWriteFooterImpl(writer->fd, writer->footer, &writer->file->size); +} + static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) { int32_t code = 0; int32_t lino = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbUpgrade.c b/source/dnode/vnode/src/tsdb/tsdbUpgrade.c index e348e60e74..de78426da1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUpgrade.c +++ b/source/dnode/vnode/src/tsdb/tsdbUpgrade.c @@ -202,11 +202,15 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray * } if (fd) { + // write footer + + // sync and close code = tsdbFsyncFile(fd); TSDB_CHECK_CODE(code, lino, _exit); - tsdbCloseFile(&fd); } + + // clear TARRAY2_DESTROY(tombBlkArray, NULL); tTombBlockDestroy(tombBlock); taosArrayDestroy(aDelData); @@ -260,10 +264,11 @@ static int32_t tsdbDoUpgradeFileSystem(STsdb *tsdb, int8_t rollback) { TFileSetArray fileSetArray[1] = {0}; - // load old file system and convert + // open old file system code = tsdbFSOpen(tsdb, rollback); TSDB_CHECK_CODE(code, lino, _exit); + // upgrade each file set for (int32_t i = 0; i < taosArrayGetSize(tsdb->fs.aDFileSet); i++) { SDFileSet *pDFileSet = taosArrayGet(tsdb->fs.aDFileSet, i); @@ -271,21 +276,23 @@ static int32_t tsdbDoUpgradeFileSystem(STsdb *tsdb, int8_t rollback) { TSDB_CHECK_CODE(code, lino, _exit); } + // upgrade tomb file if (tsdb->fs.pDelFile != NULL) { code = tsdbUpgradeTombFile(tsdb, tsdb->fs.pDelFile, fileSetArray); TSDB_CHECK_CODE(code, lino, _exit); } + // close file system code = tsdbFSClose(tsdb); TSDB_CHECK_CODE(code, lino, _exit); // save new file system char fname[TSDB_FILENAME_LEN]; current_fname(tsdb, fname, TSDB_FCURRENT); - code = save_fs(fileSetArray, fname); TSDB_CHECK_CODE(code, lino, _exit); + // clear TARRAY2_DESTROY(fileSetArray, tsdbTFileSetClear); _exit: