xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

more code

This commit is contained in:
Hongze Cheng 2023-06-02 14:39:54 +08:00
parent 884d2cc834
commit da31d92b7e
2 changed files with 352 additions and 260 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
source/dnode/vnode/src/tsdb/dev/inc/tsdbDataFileRW.h
View File

@ -26,6 +26,7 @@ extern "C" {
typedef TARRAY2(SBlockIdx) TBlockIdxArray; typedef TARRAY2(SBlockIdx) TBlockIdxArray;
typedef TARRAY2(SDataBlk) TDataBlkArray; typedef TARRAY2(SDataBlk) TDataBlkArray;
typedef TARRAY2(SColumnDataAgg) TColumnDataAggArray;
// SDataFileReader ============================================= // SDataFileReader =============================================
typedef struct SDataFileReader SDataFileReader; typedef struct SDataFileReader SDataFileReader;
@ -50,6 +51,7 @@ typedef struct SDataFileWriterConfig {
STsdb *tsdb; STsdb *tsdb;
bool hasOldFile; bool hasOldFile;
STFile of[TSDB_FTYPE_MAX]; STFile of[TSDB_FTYPE_MAX];
int8_t cmprAlg;
int32_t maxRow; int32_t maxRow;
int32_t szPage; int32_t szPage;
int64_t cid; int64_t cid;

610
source/dnode/vnode/src/tsdb/dev/tsdbDataFileRW.c
View File

@ -17,7 +17,6 @@
typedef struct { typedef struct {
SFDataPtr blockIdxPtr[1]; SFDataPtr blockIdxPtr[1];
SFDataPtr dataBlkPtr[1];
SFDataPtr rsrvd[2]; SFDataPtr rsrvd[2];
} SDataFooter; } SDataFooter;
@ -101,11 +100,12 @@ struct SDataFileWriter {
const TBlockIdxArray *blockIdxArray; const TBlockIdxArray *blockIdxArray;
int32_t blockIdxArrayIdx; int32_t blockIdxArrayIdx;
bool tbHasOldData; bool tbHasOldData;
TABLEID tbid[1];
const TDataBlkArray *dataBlkArray; const TDataBlkArray *dataBlkArray;
int32_t dataBlkArrayIdx; int32_t dataBlkArrayIdx;
SBlockData bData[1]; SBlockData bData[1];
int32_t iRow; int32_t iRow;
TABLEID tbid[1];
} ctx[1]; } ctx[1];
STFile file[TSDB_FTYPE_MAX]; STFile file[TSDB_FTYPE_MAX];
@ -119,68 +119,29 @@ struct SDataFileWriter {
STbStatisBlock sData[1]; STbStatisBlock sData[1];
}; };
int32_t tsdbDataFileWriterOpen(const SDataFileWriterConfig *config, SDataFileWriter **writer) {
writer[0] = taosMemoryCalloc(1, sizeof(SDataFileWriter));
if (!writer[0]) return TSDB_CODE_OUT_OF_MEMORY;
writer[0]->config[0] = config[0];
writer[0]->ctx->opened = false;
return 0;
}
static int32_t tsdbDataFileWriteRemainData(SDataFileWriter *writer) {
// TODO
return 0;
}
static int32_t tsdbDataFileWriteBlockIdx(SDataFileWriter *writer) { static int32_t tsdbDataFileWriteBlockIdx(SDataFileWriter *writer) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
int32_t vid = TD_VID(writer->config->tsdb->pVnode);
int64_t offset = writer->file[TSDB_FTYPE_HEAD].size; writer->footer->blockIdxPtr->offset = writer->file[TSDB_FTYPE_HEAD].size;
int64_t size = TARRAY2_DATA_LEN(writer->dataBlkArray); writer->footer->blockIdxPtr->size = TARRAY2_DATA_LEN(writer->blockIdxArray);
if (TARRAY2_SIZE(writer->blockIdxArray) > 0) {
code = tsdbWriteFile(writer->fd[TSDB_FTYPE_HEAD], offset, (void *)TARRAY2_DATA(writer->blockIdxArray), size); if (writer->footer->blockIdxPtr->size) {
code = tsdbWriteFile(writer->fd[TSDB_FTYPE_HEAD], writer->footer->blockIdxPtr->offset,
(void *)TARRAY2_DATA(writer->blockIdxArray), writer->footer->blockIdxPtr->size);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
writer->file[TSDB_FTYPE_HEAD].size += writer->footer->blockIdxPtr->size;
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(vid, lino, code); TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
}
return code;
}
static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer) {
int32_t code = 0;
int32_t lino = 0;
int32_t vid = TD_VID(writer->config->tsdb->pVnode);
code = tsdbDataFileWriteRemainData(writer);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbDataFileWriteBlockIdx(writer);
TSDB_CHECK_CODE(code, lino, _exit);
for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) {
if (writer->fd[i]) {
code = tsdbFsyncFile(writer->fd[i]);
TSDB_CHECK_CODE(code, lino, _exit);
tsdbCloseFile(&writer->fd[i]);
}
}
_exit:
if (code) {
TSDB_ERROR_LOG(vid, lino, code);
} }
return code; return code;
} }
static int32_t tsdbDataFileWriterCloseAbort(SDataFileWriter *writer) { static int32_t tsdbDataFileWriterCloseAbort(SDataFileWriter *writer) {
// TODO ASSERT(0);
return 0; return 0;
} }
@ -189,35 +150,6 @@ static int32_t tsdbDataFileWriterDoClose(SDataFileWriter *writer) {
return 0; return 0;
} }
int32_t tsdbDataFileWriterClose(SDataFileWriter **writer, bool abort, STFileOp op[/*TSDB_FTYPE_MAX*/]) {
int32_t code = 0;
int32_t lino = 0;
int32_t vid = TD_VID(writer[0]->config->tsdb->pVnode);
if (!writer[0]->ctx->opened) {
for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) {
op[i].optype = TSDB_FOP_NONE;
}
} else {
if (abort) {
code = tsdbDataFileWriterCloseAbort(writer[0]);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
code = tsdbDataFileWriterCloseCommit(writer[0]);
TSDB_CHECK_CODE(code, lino, _exit);
}
tsdbDataFileWriterDoClose(writer[0]);
}
taosMemoryFree(writer);
writer[0] = NULL;
_exit:
if (code) {
TSDB_ERROR_LOG(vid, lino, code);
}
return code;
}
static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) { static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
@ -256,27 +188,101 @@ _exit:
return code; return code;
} }
static int32_t tsdbDataFileWriteBlockData(SDataFileWriter *writer, SBlockData *bData) { static int32_t tsdbDataFileWriteDataBlock(SDataFileWriter *writer, SBlockData *bData) {
if (bData->nRow == 0) return 0; if (bData->nRow == 0) return 0;
ASSERT(bData->uid);
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
int32_t vid = TD_VID(writer->config->tsdb->pVnode);
SDataBlk dataBlk[1]; SDataBlk dataBlk[1] = {{
.minKey =
{
.ts = bData->aTSKEY[0],
.version = bData->aVersion[0],
},
.maxKey =
{
.ts = bData->aTSKEY[bData->nRow - 1],
.version = bData->aVersion[bData->nRow - 1],
},
.minVer = bData->aVersion[0],
.maxVer = bData->aVersion[0],
.nRow = bData->nRow,
.hasDup = 0,
.nSubBlock = 1,
}};
// TODO: fill dataBlk for (int32_t i = 1; i < bData->nRow; ++i) {
if (bData->aTSKEY[i] == bData->aTSKEY[i - 1]) {
dataBlk->hasDup = 1;
}
dataBlk->minVer = TMIN(dataBlk->minVer, bData->aVersion[i]);
dataBlk->maxVer = TMAX(dataBlk->maxVer, bData->aVersion[i]);
}
// TODO: write data int32_t sizeArr[5] = {0};
// to .data
code = tCmprBlockData(bData, writer->config->cmprAlg, NULL, NULL, writer->config->bufArr, sizeArr);
TSDB_CHECK_CODE(code, lino, _exit);
dataBlk->aSubBlock->offset = writer->file[TSDB_FTYPE_DATA].size;
dataBlk->aSubBlock->szKey = sizeArr[3] + sizeArr[2];
dataBlk->aSubBlock->szBlock = dataBlk->aSubBlock->szKey + sizeArr[1] + sizeArr[0];
for (int32_t i = 3; i >= 0; --i) {
if (sizeArr[i]) {
code = tsdbWriteFile(writer->fd[TSDB_FTYPE_DATA], writer->file[TSDB_FTYPE_DATA].size, writer->config->bufArr[i],
sizeArr[i]);
TSDB_CHECK_CODE(code, lino, _exit);
writer->file[TSDB_FTYPE_DATA].size += sizeArr[i];
}
}
// to .sma
TColumnDataAggArray smaArr[1] = {0};
for (int32_t i = 0; i < bData->nColData; ++i) {
SColData *colData = bData->aColData + i;
if ((!colData->smaOn) //
|| ((colData->flag & HAS_VALUE) == 0) //
) {
continue;
}
SColumnDataAgg sma[1] = {{.colId = colData->cid}};
tColDataCalcSMA[colData->type](colData, &sma->sum, &sma->max, &sma->min, &sma->numOfNull);
code = TARRAY2_APPEND_PTR(smaArr, sma);
TSDB_CHECK_CODE(code, lino, _exit);
}
dataBlk->smaInfo.offset = writer->file[TSDB_FTYPE_SMA].size;
dataBlk->smaInfo.size = TARRAY2_DATA_LEN(smaArr);
if (dataBlk->smaInfo.size) {
code = tsdbWriteFile(writer->fd[TSDB_FTYPE_SMA], dataBlk->smaInfo.offset, (const uint8_t *)TARRAY2_DATA(smaArr),
dataBlk->smaInfo.size);
TSDB_CHECK_CODE(code, lino, _exit);
writer->file[TSDB_FTYPE_SMA].size += dataBlk->smaInfo.size;
}
TARRAY2_FREE(smaArr);
// to dataBlkArray
code = TARRAY2_APPEND_PTR(writer->dataBlkArray, dataBlk); code = TARRAY2_APPEND_PTR(writer->dataBlkArray, dataBlk);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
tBlockDataClear(bData); if (bData == writer->bData) {
tBlockDataClear(writer->bData);
}
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(vid, lino, code); TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
} }
return code; return code;
} }
@ -286,7 +292,6 @@ static int32_t tsdbDataFileWriteDataBlk(SDataFileWriter *writer, const TDataBlkA
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
int32_t vid = TD_VID(writer->config->tsdb->pVnode);
SBlockIdx blockIdx[1]; SBlockIdx blockIdx[1];
blockIdx->suid = writer->ctx->tbid->suid; blockIdx->suid = writer->ctx->tbid->suid;
@ -304,7 +309,7 @@ static int32_t tsdbDataFileWriteDataBlk(SDataFileWriter *writer, const TDataBlkA
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(vid, lino, code); TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
} }
return code; return code;
} }
@ -312,20 +317,93 @@ _exit:
static int32_t tsdbDataFileDoWriteTSRow(SDataFileWriter *writer, TSDBROW *row) { static int32_t tsdbDataFileDoWriteTSRow(SDataFileWriter *writer, TSDBROW *row) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
int32_t vid = TD_VID(writer->config->tsdb->pVnode);
// update/append
if (row->type == TSDBROW_ROW_FMT) { if (row->type == TSDBROW_ROW_FMT) {
// TODO: udpate row schema code = tsdbUpdateSkmRow(writer->config->tsdb, writer->ctx->tbid, TSDBROW_SVERSION(row), writer->config->skmRow);
}
code = tBlockDataAppendRow(writer->ctx->bData, row, NULL /* TODO */, writer->ctx->tbid->uid);
TSDB_CHECK_CODE(code, lino, _exit);
if (writer->ctx->bData->nRow >= writer->config->maxRow) {
code = tsdbDataFileWriteBlockData(writer, writer->bData);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
TSDBKEY key[1] = {TSDBROW_KEY(row)};
if (key->version <= writer->config->compactVersion //
&& writer->bData->nRow > 0 //
&& writer->bData->aTSKEY[writer->bData->nRow - 1] == key->ts //
) {
code = tBlockDataUpdateRow(writer->bData, row, writer->config->skmRow->pTSchema);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
if (writer->bData->nRow >= writer->config->maxRow) {
code = tsdbDataFileWriteDataBlock(writer, writer->bData);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tBlockDataAppendRow(writer->bData, row, writer->config->skmRow->pTSchema, writer->ctx->tbid->uid);
TSDB_CHECK_CODE(code, lino, _exit);
}
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
}
return code;
}
static int32_t tsdbDataFileDoWriteTSData(SDataFileWriter *writer, TSDBROW *row) {
int32_t code = 0;
int32_t lino = 0;
int32_t vid = TD_VID(writer->config->tsdb->pVnode);
while (writer->ctx->tbHasOldData) {
for (; writer->ctx->iRow < writer->ctx->bData->nRow; writer->ctx->iRow++) {
TSDBROW row1[1] = {tsdbRowFromBlockData(writer->ctx->bData, writer->ctx->iRow)};
int32_t c = tsdbRowCmprFn(row, row1);
ASSERT(c);
if (c > 0) {
code = tsdbDataFileDoWriteTSRow(writer, row1);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
goto _do_write;
}
}
if (writer->ctx->dataBlkArrayIdx >= TARRAY2_SIZE(writer->ctx->dataBlkArray)) {
writer->ctx->tbHasOldData = false;
break;
}
for (; writer->ctx->dataBlkArrayIdx < TARRAY2_SIZE(writer->ctx->dataBlkArray); writer->ctx->dataBlkArrayIdx++) {
const SDataBlk *dataBlk = TARRAY2_GET_PTR(writer->ctx->dataBlkArray, writer->ctx->dataBlkArrayIdx);
TSDBKEY key = TSDBROW_KEY(row);
SDataBlk dataBlk1[1] = {{
.minKey = key,
.maxKey = key,
}};
int32_t c = tDataBlkCmprFn(dataBlk, dataBlk1);
if (c < 0) {
code = tsdbDataFileWriteDataBlock(writer, writer->bData);
TSDB_CHECK_CODE(code, lino, _exit);
code = TARRAY2_APPEND_PTR(writer->dataBlkArray, dataBlk);
TSDB_CHECK_CODE(code, lino, _exit);
} else if (c > 0) {
goto _do_write;
} else {
code = tsdbDataFileReadDataBlock(writer->ctx->reader, dataBlk, writer->ctx->bData);
TSDB_CHECK_CODE(code, lino, _exit);
writer->ctx->iRow = 0;
writer->ctx->dataBlkArrayIdx++;
break;
}
}
}
_do_write:
code = tsdbDataFileDoWriteTSRow(writer, row);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(vid, lino, code); TSDB_ERROR_LOG(vid, lino, code);
@ -339,6 +417,7 @@ static int32_t tsdbDataFileWriteTableDataEnd(SDataFileWriter *writer) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
// handle table remain data
if (writer->ctx->tbHasOldData) { if (writer->ctx->tbHasOldData) {
for (; writer->ctx->iRow < writer->ctx->bData->nRow; writer->ctx->iRow++) { for (; writer->ctx->iRow < writer->ctx->bData->nRow; writer->ctx->iRow++) {
TSDBROW row[1] = {tsdbRowFromBlockData(writer->ctx->bData, writer->ctx->iRow)}; TSDBROW row[1] = {tsdbRowFromBlockData(writer->ctx->bData, writer->ctx->iRow)};
@ -347,7 +426,7 @@ static int32_t tsdbDataFileWriteTableDataEnd(SDataFileWriter *writer) {
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbDataFileWriteBlockData(writer, writer->bData); code = tsdbDataFileWriteDataBlock(writer, writer->bData);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
for (; writer->ctx->dataBlkArrayIdx < TARRAY2_SIZE(writer->ctx->dataBlkArray); writer->ctx->dataBlkArrayIdx++) { for (; writer->ctx->dataBlkArrayIdx < TARRAY2_SIZE(writer->ctx->dataBlkArray); writer->ctx->dataBlkArrayIdx++) {
@ -355,9 +434,11 @@ static int32_t tsdbDataFileWriteTableDataEnd(SDataFileWriter *writer) {
TARRAY2_GET_PTR(writer->ctx->dataBlkArray, writer->ctx->dataBlkArrayIdx)); TARRAY2_GET_PTR(writer->ctx->dataBlkArray, writer->ctx->dataBlkArrayIdx));
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
writer->ctx->tbHasOldData = false;
} }
code = tsdbDataFileWriteBlockData(writer, writer->bData); code = tsdbDataFileWriteDataBlock(writer, writer->bData);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbDataFileWriteDataBlk(writer, writer->dataBlkArray); code = tsdbDataFileWriteDataBlk(writer, writer->dataBlkArray);
@ -371,154 +452,61 @@ _exit:
} }
static int32_t tsdbDataFileWriteTableDataBegin(SDataFileWriter *writer, const TABLEID *tbid) { static int32_t tsdbDataFileWriteTableDataBegin(SDataFileWriter *writer, const TABLEID *tbid) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
int32_t vid = TD_VID(writer->config->tsdb->pVnode); SMetaInfo info;
writer->ctx->tbHasOldData = false; writer->ctx->tbHasOldData = false;
for (; writer->ctx->blockIdxArrayIdx < TARRAY2_SIZE(writer->ctx->blockIdxArray); writer->ctx->blockIdxArrayIdx++) { // skip data of previous table
const SBlockIdx *blockIdx = TARRAY2_GET_PTR(writer->ctx->blockIdxArray, writer->ctx->blockIdxArrayIdx); if (writer->ctx->blockIdxArray) {
for (; writer->ctx->blockIdxArrayIdx < TARRAY2_SIZE(writer->ctx->blockIdxArray); writer->ctx->blockIdxArrayIdx++) {
const SBlockIdx *blockIdx = TARRAY2_GET_PTR(writer->ctx->blockIdxArray, writer->ctx->blockIdxArrayIdx);
int32_t c = tTABLEIDCmprFn(blockIdx, tbid); int32_t c = tTABLEIDCmprFn(blockIdx, tbid);
if (c < 0) {
SMetaInfo info;
if (metaGetInfo(writer->config->tsdb->pVnode->pMeta, blockIdx->suid, &info, NULL) == 0) {
code = tsdbDataFileReadDataBlk(writer->ctx->reader, blockIdx, &writer->ctx->dataBlkArray);
TSDB_CHECK_CODE(code, lino, _exit);
writer->ctx->tbid->suid = blockIdx->suid;
writer->ctx->tbid->uid = blockIdx->uid;
code = tsdbDataFileWriteDataBlk(writer, writer->ctx->dataBlkArray);
TSDB_CHECK_CODE(code, lino, _exit);
}
} else {
if (c == 0) {
writer->ctx->tbHasOldData = true;
code = tsdbDataFileReadDataBlk(writer->ctx->reader, blockIdx, &writer->ctx->dataBlkArray);
TSDB_CHECK_CODE(code, lino, _exit);
writer->ctx->dataBlkArrayIdx = 0;
tBlockDataReset(writer->ctx->bData);
writer->ctx->iRow = 0;
writer->ctx->blockIdxArrayIdx++;
}
break;
}
}
writer->ctx->tbid[0] = tbid[0];
_exit:
if (code) {
TSDB_ERROR_LOG(vid, lino, code);
}
return code;
}
static int32_t tsdbDataFileDoWriteTableDataRow(SDataFileWriter *writer, TSDBROW *row) {
int32_t code = 0;
int32_t lino = 0;
int32_t vid = TD_VID(writer->config->tsdb->pVnode);
while (writer->ctx->tbHasOldData) {
for (; writer->ctx->iRow < writer->ctx->bData->nRow; writer->ctx->iRow++) {
TSDBROW row1[1] = {tsdbRowFromBlockData(writer->ctx->bData, writer->ctx->iRow)};
int32_t c = tsdbRowCmprFn(row, row1);
ASSERT(c);
if (row > 0) {
code = tsdbDataFileDoWriteTSRow(writer, row1);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
goto _write_row;
}
}
for (; writer->ctx->dataBlkArrayIdx < TARRAY2_SIZE(writer->ctx->dataBlkArray); writer->ctx->dataBlkArrayIdx++) {
const SDataBlk *dataBlk = TARRAY2_GET_PTR(writer->ctx->dataBlkArray, writer->ctx->dataBlkArrayIdx);
SDataBlk dataBlk1[1] = {{.minKey = {}, .maxKey = {}}}; // TODO
int32_t c = tDataBlkCmprFn(dataBlk, dataBlk1);
if (c < 0) { if (c < 0) {
code = tsdbDataFileWriteBlockData(writer, writer->bData); if (metaGetInfo(writer->config->tsdb->pVnode->pMeta, blockIdx->uid, &info, NULL) == 0) {
TSDB_CHECK_CODE(code, lino, _exit); code = tsdbDataFileReadDataBlk(writer->ctx->reader, blockIdx, &writer->ctx->dataBlkArray);
TSDB_CHECK_CODE(code, lino, _exit);
code = TARRAY2_APPEND_PTR(writer->dataBlkArray, dataBlk); writer->ctx->tbid->suid = blockIdx->suid;
TSDB_CHECK_CODE(code, lino, _exit); writer->ctx->tbid->uid = blockIdx->uid;
} else if (c > 0) {
goto _write_row; code = tsdbDataFileWriteDataBlk(writer, writer->ctx->dataBlkArray);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
continue;
}
} else { } else {
code = tsdbDataFileReadDataBlock(writer->ctx->reader, dataBlk, writer->ctx->bData); if (c == 0) {
TSDB_CHECK_CODE(code, lino, _exit); writer->ctx->tbHasOldData = true;
writer->ctx->iRow = 0;
writer->ctx->dataBlkArrayIdx++; code = tsdbDataFileReadDataBlk(writer->ctx->reader, blockIdx, &writer->ctx->dataBlkArray);
TSDB_CHECK_CODE(code, lino, _exit);
writer->ctx->dataBlkArrayIdx = 0;
tBlockDataReset(writer->ctx->bData);
writer->ctx->iRow = 0;
writer->ctx->blockIdxArrayIdx++;
}
break; break;
} }
} }
if (writer->ctx->dataBlkArrayIdx >= TARRAY2_SIZE(writer->ctx->dataBlkArray) //
&& writer->ctx->iRow >= writer->ctx->bData->nRow) {
writer->ctx->tbHasOldData = false;
}
} }
_write_row: // make sure state is correct
code = tsdbDataFileDoWriteTSRow(writer, row); writer->ctx->tbid[0] = tbid[0];
if (tbid->suid == INT64_MAX && tbid->uid == INT64_MAX) goto _exit;
TARRAY2_CLEAR(writer->dataBlkArray, NULL);
code = tsdbUpdateSkmTb(writer->config->tsdb, tbid, writer->config->skmTb);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
_exit: code = tBlockDataInit(writer->bData, writer->ctx->tbid, writer->config->skmTb->pTSchema, NULL, 0);
if (code) {
TSDB_ERROR_LOG(vid, lino, code);
}
return code;
}
static int32_t tsdbDataFileDoWriteTableDataBlock(SDataFileWriter *writer, SBlockData *bData) {
int32_t code = 0;
int32_t lino = 0;
int32_t vid = TD_VID(writer->config->tsdb->pVnode);
if (!writer->ctx->tbHasOldData && writer->bData->nRow == 0) {
code = tsdbDataFileWriteBlockData(writer, bData);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
for (int32_t i = 0; i < bData->nRow; i++) {
TSDBROW row[1] = {tsdbRowFromBlockData(bData, i)};
code = tsdbDataFileDoWriteTableDataRow(writer, row);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
_exit:
if (code) {
TSDB_ERROR_LOG(vid, lino, code);
}
return code;
}
int32_t tsdbDataFileWriteTSDataBlock(SDataFileWriter *writer, SBlockData *bData) {
if (bData->nRow == 0) return 0;
int32_t code = 0;
int32_t lino = 0;
ASSERT(bData->uid);
if (!writer->ctx->opened) {
code = tsdbDataFileWriterDoOpen(writer);
TSDB_CHECK_CODE(code, lino, _exit);
}
if (bData->uid != writer->ctx->tbid->uid) {
code = tsdbDataFileWriteTableDataEnd(writer);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbDataFileWriteTableDataBegin(writer, (TABLEID *)bData);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbDataFileDoWriteTableDataBlock(writer, bData);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
@ -528,6 +516,100 @@ _exit:
return code; return code;
} }
static int32_t tsdbDataFileWriteFooter(SDataFileWriter *writer) {
int32_t code = 0;
int32_t lino = 0;
code = tsdbWriteFile(writer->fd[TSDB_FTYPE_HEAD], writer->file[TSDB_FTYPE_HEAD].size, (const uint8_t *)writer->footer,
sizeof(SDataFooter));
TSDB_CHECK_CODE(code, lino, _exit);
writer->file[TSDB_FTYPE_HEAD].size += sizeof(SDataFooter);
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
}
return code;
}
static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, STFileOp *op) {
int32_t code = 0;
int32_t lino = 0;
TABLEID tbid[1] = {{INT64_MAX, INT64_MAX}};
code = tsdbDataFileWriteTableDataEnd(writer);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbDataFileWriteTableDataBegin(writer, tbid);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbDataFileWriteBlockIdx(writer);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbDataFileWriteFooter(writer);
TSDB_CHECK_CODE(code, lino, _exit);
for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) {
if (writer->fd[i]) {
code = tsdbFsyncFile(writer->fd[i]);
TSDB_CHECK_CODE(code, lino, _exit);
tsdbCloseFile(&writer->fd[i]);
}
}
// .head
// .data
// .sma
// .tomb
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
}
return code;
}
int32_t tsdbDataFileWriterOpen(const SDataFileWriterConfig *config, SDataFileWriter **writer) {
writer[0] = taosMemoryCalloc(1, sizeof(SDataFileWriter));
if (!writer[0]) return TSDB_CODE_OUT_OF_MEMORY;
writer[0]->config[0] = config[0];
writer[0]->ctx->opened = false;
return 0;
}
int32_t tsdbDataFileWriterClose(SDataFileWriter **writer, bool abort, STFileOp op[/*TSDB_FTYPE_MAX*/]) {
int32_t code = 0;
int32_t lino = 0;
if (!writer[0]->ctx->opened) {
for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) {
op[i].optype = TSDB_FOP_NONE;
}
} else {
if (abort) {
code = tsdbDataFileWriterCloseAbort(writer[0]);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
code = tsdbDataFileWriterCloseCommit(writer[0], op);
TSDB_CHECK_CODE(code, lino, _exit);
}
tsdbDataFileWriterDoClose(writer[0]);
}
taosMemoryFree(writer);
writer[0] = NULL;
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(writer[0]->config->tsdb->pVnode), lino, code);
}
return code;
}
int32_t tsdbDataFileWriteTSData(SDataFileWriter *writer, SRowInfo *row) { int32_t tsdbDataFileWriteTSData(SDataFileWriter *writer, SRowInfo *row) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
@ -541,36 +623,43 @@ int32_t tsdbDataFileWriteTSData(SDataFileWriter *writer, SRowInfo *row) {
code = tsdbDataFileWriteTableDataEnd(writer); code = tsdbDataFileWriteTableDataEnd(writer);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
writer->ctx->tbid->suid = row->uid;
writer->ctx->tbid->uid = row->uid;
code = tsdbDataFileWriteTableDataBegin(writer, (TABLEID *)row); code = tsdbDataFileWriteTableDataBegin(writer, (TABLEID *)row);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
if (row->row.type == TSDBROW_ROW_FMT) { code = tsdbDataFileDoWriteTSData(writer, &row->row);
code = tsdbUpdateSkmRow(writer->config->tsdb, writer->ctx->tbid, // TSDB_CHECK_CODE(code, lino, _exit);
TSDBROW_SVERSION(&row->row), writer->config->skmRow);
TSDB_CHECK_CODE(code, lino, _exit);
}
// update/append _exit:
TSDBKEY key[1] = {TSDBROW_KEY(&row->row)}; if (code) {
if (key->version <= writer->config->compactVersion // TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
&& writer->bData->nRow > 0 //
&& writer->bData->aTSKEY[writer->bData->nRow - 1] == key->ts //
) {
code = tBlockDataUpdateRow(writer->bData, &row->row, writer->config->skmRow->pTSchema);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
if (writer->bData->nRow >= writer->config->maxRow) {
code = tsdbDataFileDoWriteTableDataBlock(writer, writer->bData);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tBlockDataAppendRow(writer->bData, &row->row, writer->config->skmRow->pTSchema, row->uid);
TSDB_CHECK_CODE(code, lino, _exit);
} }
return code;
}
int32_t tsdbDataFileWriteTSDataBlock(SDataFileWriter *writer, SBlockData *bData) {
if (bData->nRow == 0) return 0;
int32_t code = 0;
int32_t lino = 0;
// ASSERT(bData->uid);
// if (!writer->ctx->opened) {
// code = tsdbDataFileWriterDoOpen(writer);
// TSDB_CHECK_CODE(code, lino, _exit);
// }
// if (bData->uid != writer->ctx->tbid->uid) {
// code = tsdbDataFileWriteTableDataEnd(writer);
// TSDB_CHECK_CODE(code, lino, _exit);
// code = tsdbDataFileWriteTableDataBegin(writer, (TABLEID *)bData);
// TSDB_CHECK_CODE(code, lino, _exit);
// }
// code = tsdbDataFileDoWriteTableDataBlock(writer, bData);
// TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
@ -580,6 +669,7 @@ _exit:
} }
int32_t tsdbDataFileFLushTSDataBlock(SDataFileWriter *writer) { int32_t tsdbDataFileFLushTSDataBlock(SDataFileWriter *writer) {
if (writer->bData->nRow == 0) return 0; // if (writer->bData->nRow == 0) return 0;
return tsdbDataFileDoWriteTableDataBlock(writer, writer->bData); // return tsdbDataFileDoWriteTableDataBlock(writer, writer->bData);
} return 0;
}