xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refact more code

This commit is contained in:
Hongze Cheng 2023-06-19 09:32:29 +08:00
parent aba73614df
commit 073a2b8dd3
5 changed files with 121 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

111
source/dnode/vnode/src/tsdb/tsdbCommit2.c
View File

@ -21,8 +21,8 @@ typedef struct {
TFileSetArray *fsetArr;
TFileOpArray fopArray[1];
SSkmInfo skmTb[1];
SSkmInfo skmRow[1];
// SSkmInfo skmTb[1];
// SSkmInfo skmRow[1];
int32_t minutes;
int8_t precision;
@ -56,45 +56,29 @@ typedef struct {
SIterMerger *tombIterMerger;
// writer
SBlockData blockData[2];
int32_t blockDataIdx;
SDataFileWriter *dataWriter;
SSttFileWriter *sttWriter;
SFSetWriter *writer;
} SCommitter2;
static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) {
int32_t code = 0;
int32_t lino = 0;
// stt writer
SSttFileWriterConfig config[1] = {{
SFSetWriterConfig config = {
.tsdb = committer->tsdb,
.toSttOnly = true,
.compactVersion = committer->compactVersion,
.minRow = committer->minRow,
.maxRow = committer->maxRow,
.szPage = committer->szPage,
.cmprAlg = committer->cmprAlg,
.compactVersion = committer->compactVersion,
.did = committer->ctx->did,
.fid = committer->ctx->fid,
.cid = committer->ctx->cid,
.did = committer->ctx->did,
.level = 0,
}};
};
code = tsdbSttFileWriterOpen(config, &committer->sttWriter);
TSDB_CHECK_CODE(code, lino, _exit);
// data writer
if (committer->sttTrigger == 1) {
// data writer
SDataFileWriterConfig config = {
.tsdb = committer->tsdb,
.cmprAlg = committer->cmprAlg,
.maxRow = committer->maxRow,
.szPage = committer->szPage,
.fid = committer->ctx->fid,
.cid = committer->ctx->cid,
.did = committer->ctx->did,
.compactVersion = committer->compactVersion,
};
config.toSttOnly = false;
if (committer->ctx->fset) {
for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ftype++) {
@ -104,11 +88,11 @@ static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) {
}
}
}
code = tsdbDataFileWriterOpen(&config, &committer->dataWriter);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbFSetWriterOpen(&config, &committer->writer);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code);
@ -117,22 +101,10 @@ _exit:
}
static int32_t tsdbCommitCloseWriter(SCommitter2 *committer) {
int32_t code = 0;
int32_t lino = 0;
code = tsdbSttFileWriterClose(&committer->sttWriter, 0, committer->fopArray);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbDataFileWriterClose(&committer->dataWriter, 0, committer->fopArray);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code);
}
return code;
return tsdbFSetWriterClose(&committer->writer, 0, committer->fopArray);
}
#if 0
static int32_t tsdbCommitTSDataToDataTableBegin(SCommitter2 *committer, const TABLEID *tbid) {
int32_t code = 0;
int32_t lino = 0;
@ -328,17 +300,30 @@ _exit:
}
return code;
}
#endif
static int32_t tsdbCommitTSData(SCommitter2 *committer) {
int32_t code = 0;
int32_t lino = 0;
int32_t code = 0;
int32_t lino = 0;
SMetaInfo info;
// loop iter
if (committer->sttTrigger == 1) {
code = tsdbCommitTSDataToData(committer);
for (SRowInfo *row; (row = tsdbIterMergerGetData(committer->dataIterMerger)) != NULL;) {
if (row->uid != committer->ctx->tbid->uid) {
// Ignore table of obsolescence
if (metaGetInfo(committer->tsdb->pVnode->pMeta, row->uid, &info, NULL) != 0) {
code = tsdbIterMergerSkipTableData(committer->dataIterMerger, (TABLEID *)row);
TSDB_CHECK_CODE(code, lino, _exit);
continue;
}
committer->ctx->tbid->suid = row->suid;
committer->ctx->tbid->uid = row->uid;
}
code = tsdbFSetWriteRow(committer->writer, row);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
code = tsdbCommitTSDataToStt(committer);
code = tsdbIterMergerNext(committer->dataIterMerger);
TSDB_CHECK_CODE(code, lino, _exit);
}
@ -353,22 +338,12 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) {
int32_t code = 0;
int32_t lino = 0;
if (committer->dataWriter == NULL || tsdbSttFileWriterIsOpened(committer->sttWriter)) {
for (STombRecord *record; (record = tsdbIterMergerGetTombRecord(committer->tombIterMerger));) {
code = tsdbSttFileWriteTombRecord(committer->sttWriter, record);
TSDB_CHECK_CODE(code, lino, _exit);
for (STombRecord *record; (record = tsdbIterMergerGetTombRecord(committer->tombIterMerger));) {
code = tsdbFSetWriteTombRecord(committer->writer, record);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbIterMergerNext(committer->tombIterMerger);
TSDB_CHECK_CODE(code, lino, _exit);
}
} else {
for (STombRecord *record; (record = tsdbIterMergerGetTombRecord(committer->tombIterMerger));) {
code = tsdbDataFileWriteTombRecord(committer->dataWriter, record);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbIterMergerNext(committer->tombIterMerger);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbIterMergerNext(committer->tombIterMerger);
TSDB_CHECK_CODE(code, lino, _exit);
}
_exit:
@ -529,8 +504,7 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) {
ASSERT(TARRAY2_SIZE(committer->dataIterArray) == 0);
ASSERT(committer->dataIterMerger == NULL);
ASSERT(committer->sttWriter == NULL);
ASSERT(committer->dataWriter == NULL);
ASSERT(committer->writer == NULL);
code = tsdbCommitOpenReader(committer);
TSDB_CHECK_CODE(code, lino, _exit);
@ -660,8 +634,7 @@ static int32_t tsdbCloseCommitter(SCommitter2 *committer, int32_t eno) {
ASSERT(0);
}
ASSERT(committer->dataWriter == NULL);
ASSERT(committer->sttWriter == NULL);
ASSERT(committer->writer == NULL);
ASSERT(committer->dataIterMerger == NULL);
ASSERT(committer->tombIterMerger == NULL);
TARRAY2_DESTROY(committer->dataIterArray, NULL);

1
source/dnode/vnode/src/tsdb/tsdbCommit2.h
View File

@ -15,6 +15,7 @@
#include "tsdbDataFileRW.h"
#include "tsdbFS2.h"
#include "tsdbFSetRW.h"
#include "tsdbIter.h"
#include "tsdbSttFileRW.h"

1
source/dnode/vnode/src/tsdb/tsdbFSetRW.h
View File

@ -41,7 +41,6 @@ typedef struct {
bool exist;
STFile file;
} files[TSDB_FTYPE_MAX];
STFile sttFile;
} SFSetWriterConfig;
int32_t tsdbFSetWriterOpen(SFSetWriterConfig *config, SFSetWriter **writer);

161
source/dnode/vnode/src/tsdb/tsdbMerge.c
View File

@ -19,15 +19,13 @@ typedef struct {
STsdb *tsdb;
TFileSetArray *fsetArr;
int32_t sttTrigger;
int32_t maxRow;
int32_t minRow;
int32_t szPage;
int8_t cmprAlg;
int64_t compactVersion;
int64_t cid;
SSkmInfo skmTb[1];
SSkmInfo skmRow[1];
int32_t sttTrigger;
int32_t maxRow;
int32_t minRow;
int32_t szPage;
int8_t cmprAlg;
int64_t compactVersion;
int64_t cid;
// context
struct {
@ -37,10 +35,7 @@ typedef struct {
bool toData;
int32_t level;
SSttLvl *lvl;
// STFileObj *fobj;
TABLEID tbid[1];
int32_t blockDataIdx;
SBlockData blockData[2];
} ctx[1];
TFileOpArray fopArr[1];
@ -53,8 +48,7 @@ typedef struct {
TTsdbIterArray tombIterArr[1];
SIterMerger *tombIterMerger;
// writer
SSttFileWriter *sttWriter;
SDataFileWriter *dataWriter;
SFSetWriter *writer;
} SMerger;
static int32_t tsdbMergerOpen(SMerger *merger) {
@ -86,8 +80,7 @@ static int32_t tsdbMergerClose(SMerger *merger) {
}
taosThreadRwlockUnlock(&merger->tsdb->rwLock);
ASSERT(merger->dataWriter == NULL);
ASSERT(merger->sttWriter == NULL);
ASSERT(merger->writer == NULL);
ASSERT(merger->dataIterMerger == NULL);
ASSERT(TARRAY2_SIZE(merger->dataIterArr) == 0);
ASSERT(TARRAY2_SIZE(merger->sttReaderArr) == 0);
@ -96,11 +89,6 @@ static int32_t tsdbMergerClose(SMerger *merger) {
TARRAY2_DESTROY(merger->dataIterArr, NULL);
TARRAY2_DESTROY(merger->sttReaderArr, NULL);
TARRAY2_DESTROY(merger->fopArr, NULL);
for (int32_t i = 0; i < ARRAY_SIZE(merger->ctx->blockData); i++) {
tBlockDataDestroy(merger->ctx->blockData + i);
}
tDestroyTSchema(merger->skmTb->pTSchema);
tDestroyTSchema(merger->skmRow->pTSchema);
_exit:
if (code) {
@ -109,6 +97,7 @@ _exit:
return code;
}
#if 0
static int32_t tsdbMergeToDataTableEnd(SMerger *merger) {
if (merger->ctx->blockData[0].nRow + merger->ctx->blockData[1].nRow == 0) return 0;
@ -297,6 +286,7 @@ _exit:
}
return code;
}
#endif
static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) {
int32_t code = 0;
@ -413,49 +403,36 @@ static int32_t tsdbMergeFileSetBeginOpenWriter(SMerger *merger) {
code = TSDB_CODE_FS_NO_VALID_DISK;
TSDB_CHECK_CODE(code, lino, _exit);
}
{
// to new level
SSttFileWriterConfig config[1] = {{
.tsdb = merger->tsdb,
.maxRow = merger->maxRow,
.szPage = merger->szPage,
.cmprAlg = merger->cmprAlg,
.compactVersion = merger->compactVersion,
.did = did,
.fid = merger->ctx->fset->fid,
.cid = merger->cid,
.level = merger->ctx->level,
}};
code = tsdbSttFileWriterOpen(config, &merger->sttWriter);
TSDB_CHECK_CODE(code, lino, _exit);
}
SFSetWriterConfig config = {
.tsdb = merger->tsdb,
.toSttOnly = true,
.compactVersion = merger->compactVersion,
.minRow = merger->minRow,
.maxRow = merger->maxRow,
.szPage = merger->szPage,
.cmprAlg = merger->cmprAlg,
.fid = merger->ctx->fset->fid,
.cid = merger->cid,
.did = did,
.level = merger->ctx->level,
};
if (merger->ctx->toData) {
SDataFileWriterConfig config[1] = {{
.tsdb = merger->tsdb,
.cmprAlg = merger->cmprAlg,
.maxRow = merger->maxRow,
.szPage = merger->szPage,
.fid = merger->ctx->fset->fid,
.cid = merger->cid,
.did = did,
.compactVersion = merger->compactVersion,
}};
config.toSttOnly = false;
for (int32_t i = 0; i < TSDB_FTYPE_MAX; i++) {
if (merger->ctx->fset->farr[i]) {
config->files[i].exist = true;
config->files[i].file = merger->ctx->fset->farr[i]->f[0];
for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ++ftype) {
if (merger->ctx->fset->farr[ftype]) {
config.files[ftype].exist = true;
config.files[ftype].file = merger->ctx->fset->farr[ftype]->f[0];
} else {
config->files[i].exist = false;
config.files[ftype].exist = false;
}
}
code = tsdbDataFileWriterOpen(config, &merger->dataWriter);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbFSetWriterOpen(&config, &merger->writer);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
TSDB_ERROR_LOG(vid, lino, code);
@ -470,15 +447,10 @@ static int32_t tsdbMergeFileSetBegin(SMerger *merger) {
ASSERT(TARRAY2_SIZE(merger->sttReaderArr) == 0);
ASSERT(TARRAY2_SIZE(merger->dataIterArr) == 0);
ASSERT(merger->dataIterMerger == NULL);
ASSERT(merger->sttWriter == NULL);
ASSERT(merger->dataWriter == NULL);
ASSERT(merger->writer == NULL);
merger->ctx->tbid->suid = 0;
merger->ctx->tbid->uid = 0;
merger->ctx->blockDataIdx = 0;
for (int32_t i = 0; i < ARRAY_SIZE(merger->ctx->blockData); ++i) {
tBlockDataReset(merger->ctx->blockData + i);
}
// open reader
code = tsdbMergeFileSetBeginOpenReader(merger);
@ -500,23 +472,7 @@ _exit:
}
static int32_t tsdbMergeFileSetEndCloseWriter(SMerger *merger) {
int32_t code = 0;
int32_t lino = 0;
int32_t vid = TD_VID(merger->tsdb->pVnode);
code = tsdbSttFileWriterClose(&merger->sttWriter, 0, merger->fopArr);
TSDB_CHECK_CODE(code, lino, _exit);
if (merger->ctx->toData) {
code = tsdbDataFileWriterClose(&merger->dataWriter, 0, merger->fopArr);
TSDB_CHECK_CODE(code, lino, _exit);
}
_exit:
if (code) {
TSDB_ERROR_LOG(vid, lino, code);
}
return code;
return tsdbFSetWriterClose(&merger->writer, 0, merger->fopArr);
}
static int32_t tsdbMergeFileSetEndCloseIter(SMerger *merger) {
@ -560,12 +516,49 @@ static int32_t tsdbMergeFileSet(SMerger *merger, STFileSet *fset) {
code = tsdbMergeFileSetBegin(merger);
TSDB_CHECK_CODE(code, lino, _exit);
// do merge
if (merger->ctx->toData) {
code = tsdbMergeToDataLevel(merger);
// data
SMetaInfo info;
SRowInfo *row;
merger->ctx->tbid->suid = 0;
merger->ctx->tbid->uid = 0;
while ((row = tsdbIterMergerGetData(merger->dataIterMerger)) != NULL) {
if (row->uid != merger->ctx->tbid->uid) {
if (metaGetInfo(merger->tsdb->pVnode->pMeta, row->uid, &info, NULL) != 0) {
code = tsdbIterMergerSkipTableData(merger->dataIterMerger, (TABLEID *)row);
TSDB_CHECK_CODE(code, lino, _exit);
continue;
}
merger->ctx->tbid->uid = row->uid;
merger->ctx->tbid->suid = row->suid;
}
code = tsdbFSetWriteRow(merger->writer, row);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
code = tsdbMergeToUpperLevel(merger);
code = tsdbIterMergerNext(merger->dataIterMerger);
TSDB_CHECK_CODE(code, lino, _exit);
}
// tomb
STombRecord *record;
merger->ctx->tbid->suid = 0;
merger->ctx->tbid->uid = 0;
while ((record = tsdbIterMergerGetTombRecord(merger->tombIterMerger)) != NULL) {
if (record->uid != merger->ctx->tbid->uid) {
merger->ctx->tbid->uid = record->uid;
merger->ctx->tbid->suid = record->suid;
if (metaGetInfo(merger->tsdb->pVnode->pMeta, record->uid, &info, NULL) != 0) {
code = tsdbIterMergerSkipTableData(merger->tombIterMerger, merger->ctx->tbid);
TSDB_CHECK_CODE(code, lino, _exit);
continue;
}
}
code = tsdbFSetWriteTombRecord(merger->writer, record);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbIterMergerNext(merger->tombIterMerger);
TSDB_CHECK_CODE(code, lino, _exit);
}

1
source/dnode/vnode/src/tsdb/tsdbMerge.h
View File

@ -15,6 +15,7 @@
#include "tsdbDataFileRW.h"
#include "tsdbFS2.h"
#include "tsdbFSetRW.h"
#include "tsdbIter.h"
#include "tsdbSttFileRW.h"
#include "tsdbUtil2.h"