more code

This commit is contained in:
Hongze Cheng 2023-06-01 15:08:45 +08:00
parent bdc9fc473c
commit e99b33ede5
10 changed files with 164 additions and 139 deletions

View File

@ -39,7 +39,7 @@ int32_t tsdbCloseFS(STFileSystem **fs);
int32_t tsdbFSCreateCopySnapshot(STFileSystem *fs, TFileSetArray **fsetArr); int32_t tsdbFSCreateCopySnapshot(STFileSystem *fs, TFileSetArray **fsetArr);
int32_t tsdbFSDestroyCopySnapshot(TFileSetArray **fsetArr); int32_t tsdbFSDestroyCopySnapshot(TFileSetArray **fsetArr);
// txn // txn
int32_t tsdbFSAllocEid(STFileSystem *fs, int64_t *eid); int64_t tsdbFSAllocEid(STFileSystem *fs);
int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype); int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype);
int32_t tsdbFSEditCommit(STFileSystem *fs); int32_t tsdbFSEditCommit(STFileSystem *fs);
int32_t tsdbFSEditAbort(STFileSystem *fs); int32_t tsdbFSEditAbort(STFileSystem *fs);

View File

@ -52,7 +52,7 @@ int32_t tsdbTFileSetApplyEdit(STsdb *pTsdb, const STFileSet *fset1, STFileSet *f
// max commit id // max commit id
int64_t tsdbTFileSetMaxCid(const STFileSet *fset); int64_t tsdbTFileSetMaxCid(const STFileSet *fset);
// get // get
SSttLvl *tsdbTFileSetGetLvl(STFileSet *fset, int32_t level); SSttLvl *tsdbTFileSetGetSttLvl(STFileSet *fset, int32_t level);
// is empty // is empty
bool tsdbTFileSetIsEmpty(const STFileSet *fset); bool tsdbTFileSetIsEmpty(const STFileSet *fset);

View File

@ -53,8 +53,8 @@ int32_t tsdbIterClose(STsdbIter **iter);
int32_t tsdbIterNext(STsdbIter *iter); int32_t tsdbIterNext(STsdbIter *iter);
// SIterMerger =============== // SIterMerger ===============
int32_t tsdbIterMergerInit(const TTsdbIterArray *iterArray, SIterMerger **merger); int32_t tsdbIterMergerOpen(const TTsdbIterArray *iterArray, SIterMerger **merger);
int32_t tsdbIterMergerClear(SIterMerger **merger); int32_t tsdbIterMergerClose(SIterMerger **merger);
int32_t tsdbIterMergerNext(SIterMerger *merger); int32_t tsdbIterMergerNext(SIterMerger *merger);
SRowInfo *tsdbIterMergerGet(SIterMerger *merger); SRowInfo *tsdbIterMergerGet(SIterMerger *merger);
int32_t tsdbIterMergerSkipTableData(SIterMerger *merger, const TABLEID *tbid); int32_t tsdbIterMergerSkipTableData(SIterMerger *merger, const TABLEID *tbid);

View File

@ -52,8 +52,6 @@ struct SSttFileReaderConfig {
STsdb *tsdb; STsdb *tsdb;
int32_t szPage; int32_t szPage;
STFile file[1]; STFile file[1];
SSkmInfo *skmTb;
SSkmInfo *skmRow;
uint8_t **bufArr; uint8_t **bufArr;
}; };

View File

@ -18,15 +18,19 @@
// extern dependencies // extern dependencies
typedef struct { typedef struct {
STsdb *tsdb; STsdb *tsdb;
TFileSetArray *fsetArr;
int32_t minutes; int32_t minutes;
int8_t precision; int8_t precision;
int32_t minRow; int32_t minRow;
int32_t maxRow; int32_t maxRow;
int8_t cmprAlg; int8_t cmprAlg;
int8_t sttTrigger; int32_t sttTrigger;
int32_t szPage;
int64_t compactVersion; int64_t compactVersion;
struct { struct {
int64_t cid;
int64_t now; int64_t now;
TSKEY nextKey; TSKEY nextKey;
int32_t fid; int32_t fid;
@ -37,111 +41,115 @@ typedef struct {
TABLEID tbid[1]; TABLEID tbid[1];
} ctx[1]; } ctx[1];
int64_t eid; // edit id
TFileOpArray fopArray[1]; TFileOpArray fopArray[1];
TTsdbIterArray iterArray[1]; TTsdbIterArray iterArray[1];
SIterMerger *iterMerger; SIterMerger *iterMerger;
// writer // writer
SDataFileWriter *dataWriter;
SSttFileWriter *sttWriter; SSttFileWriter *sttWriter;
SDataFileWriter *dataWriter;
} SCommitter2; } SCommitter2;
static int32_t tsdbCommitOpenNewSttWriter(SCommitter2 *pCommitter) { static int32_t tsdbCommitOpenNewSttWriter(SCommitter2 *committer) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
STsdb *pTsdb = pCommitter->tsdb;
SVnode *pVnode = pTsdb->pVnode;
int32_t vid = TD_VID(pVnode);
SSttFileWriterConfig config[1];
SDiskID did[1]; SDiskID did[1];
if (tfsAllocDisk(committer->tsdb->pVnode->pTfs, committer->ctx->expLevel, did) < 0) {
if (tfsAllocDisk(pVnode->pTfs, pCommitter->ctx->expLevel, did) < 0) {
code = TSDB_CODE_FS_NO_VALID_DISK; code = TSDB_CODE_FS_NO_VALID_DISK;
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
config->tsdb = pTsdb; SSttFileWriterConfig config[1] = {{
config->maxRow = pCommitter->maxRow; .tsdb = committer->tsdb,
config->szPage = pVnode->config.tsdbPageSize; .maxRow = committer->maxRow,
config->cmprAlg = pCommitter->cmprAlg; .szPage = committer->tsdb->pVnode->config.tsdbPageSize,
config->skmTb = NULL; .cmprAlg = committer->cmprAlg,
config->skmRow = NULL; .compactVersion = committer->compactVersion,
config->aBuf = NULL; .file =
config->file.type = TSDB_FTYPE_STT; {
config->file.did = did[0]; .type = TSDB_FTYPE_STT,
config->file.fid = pCommitter->ctx->fid; .did = did[0],
config->file.cid = pCommitter->eid; .fid = committer->ctx->fid,
config->file.size = 0; .cid = committer->ctx->cid,
config->file.stt->level = 0; },
config->file.stt->nseg = 0; }};
code = tsdbSttFileWriterOpen(config, &pCommitter->sttWriter); code = tsdbSttFileWriterOpen(config, &committer->sttWriter);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", vid, __func__, lino, tstrerror(code)); TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code);
} else { } else {
tsdbDebug("vgId:%d %s success", vid, __func__); tsdbDebug("vgId:%d %s success", TD_VID(committer->tsdb->pVnode), __func__);
} }
return code; return code;
} }
static int32_t tsdbCommitOpenExistSttWriter(SCommitter2 *pCommitter, const STFile *pFile) {
static int32_t tsdbCommitOpenExistSttWriter(SCommitter2 *committer, const STFile *f) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
STsdb *pTsdb = pCommitter->tsdb;
SVnode *pVnode = pTsdb->pVnode;
int32_t vid = TD_VID(pVnode);
SSttFileWriterConfig config = { SSttFileWriterConfig config[1] = {{
// .tsdb = committer->tsdb,
.tsdb = pTsdb, .maxRow = committer->maxRow,
.maxRow = pCommitter->maxRow, .szPage = committer->szPage,
.szPage = pVnode->config.tsdbPageSize, .cmprAlg = committer->cmprAlg,
.cmprAlg = pCommitter->cmprAlg, .compactVersion = committer->compactVersion,
.skmTb = NULL, .file = f[0],
.skmRow = NULL, }};
.aBuf = NULL,
.file = *pFile //
};
code = tsdbSttFileWriterOpen(&config, &pCommitter->sttWriter); code = tsdbSttFileWriterOpen(config, &committer->sttWriter);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", vid, __func__, lino, tstrerror(code)); TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code);
} else { } else {
tsdbDebug("vgId:%d %s success", vid, __func__); tsdbDebug("vgId:%d %s success", TD_VID(committer->tsdb->pVnode), __func__);
} }
return code; return code;
} }
static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) { static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) {
int32_t code = 0;
int32_t lino = 0;
// stt writer
if (!committer->ctx->fset) { if (!committer->ctx->fset) {
return tsdbCommitOpenNewSttWriter(committer); code = tsdbCommitOpenNewSttWriter(committer);
TSDB_CHECK_CODE(code, lino, _exit);
} }
const SSttLvl *lvl0 = tsdbTFileSetGetLvl(committer->ctx->fset, 0); const SSttLvl *lvl0 = tsdbTFileSetGetSttLvl(committer->ctx->fset, 0);
if (lvl0 == NULL) { if (lvl0 == NULL || TARRAY2_SIZE(lvl0->fobjArr) == 0) {
return tsdbCommitOpenNewSttWriter(committer); code = tsdbCommitOpenNewSttWriter(committer);
TSDB_CHECK_CODE(code, lino, _exit);
} }
ASSERT(TARRAY2_SIZE(lvl0->fobjArr) > 0);
STFileObj *fobj = TARRAY2_LAST(lvl0->fobjArr); STFileObj *fobj = TARRAY2_LAST(lvl0->fobjArr);
if (fobj->f->stt->nseg >= committer->sttTrigger) { if (fobj->f->stt->nseg >= committer->sttTrigger) {
return tsdbCommitOpenNewSttWriter(committer); code = tsdbCommitOpenNewSttWriter(committer);
TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
return tsdbCommitOpenExistSttWriter(committer, fobj->f); code = tsdbCommitOpenExistSttWriter(committer, fobj->f);
TSDB_CHECK_CODE(code, lino, _exit);
} }
// data writer
if (0) {
// TODO
}
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code);
}
return code;
} }
static int32_t tsdbCommitTSRow(SCommitter2 *committer, SRowInfo *row) { static int32_t tsdbCommitWriteDelData(SCommitter2 *committer, int64_t suid, int64_t uid, int64_t version, int64_t sKey,
return tsdbSttFileWriteTSData(committer->sttWriter, row);
}
static int32_t tsdbCommitWriteDelData(SCommitter2 *pCommitter, int64_t suid, int64_t uid, int64_t version, int64_t sKey,
int64_t eKey) { int64_t eKey) {
int32_t code = 0; int32_t code = 0;
// TODO // TODO
@ -174,7 +182,7 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) {
code = TARRAY2_APPEND(committer->iterArray, iter); code = TARRAY2_APPEND(committer->iterArray, iter);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbIterMergerInit(committer->iterArray, &committer->iterMerger); code = tsdbIterMergerOpen(committer->iterArray, &committer->iterMerger);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// loop iter // loop iter
@ -183,7 +191,7 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) {
committer->ctx->tbid->suid = row->suid; committer->ctx->tbid->suid = row->suid;
committer->ctx->tbid->uid = row->uid; committer->ctx->tbid->uid = row->uid;
// Ignore deleted table // Ignore table of obsolescence
SMetaInfo info[1]; SMetaInfo info[1];
if (metaGetInfo(committer->tsdb->pVnode->pMeta, row->uid, info, NULL) != 0) { if (metaGetInfo(committer->tsdb->pVnode->pMeta, row->uid, info, NULL) != 0) {
code = tsdbIterMergerSkipTableData(committer->iterMerger, committer->ctx->tbid); code = tsdbIterMergerSkipTableData(committer->iterMerger, committer->ctx->tbid);
@ -195,11 +203,10 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) {
TSKEY ts = TSDBROW_TS(&row->row); TSKEY ts = TSDBROW_TS(&row->row);
if (ts > committer->ctx->maxKey) { if (ts > committer->ctx->maxKey) {
committer->ctx->nextKey = TMIN(committer->ctx->nextKey, ts); committer->ctx->nextKey = TMIN(committer->ctx->nextKey, ts);
code = tsdbIterMergerSkipTableData(committer->iterMerger, committer->ctx->tbid); code = tsdbIterMergerSkipTableData(committer->iterMerger, committer->ctx->tbid);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
code = tsdbCommitTSRow(committer, row); code = tsdbSttFileWriteTSData(committer->sttWriter, row);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbIterMergerNext(committer->iterMerger); code = tsdbIterMergerNext(committer->iterMerger);
@ -216,7 +223,7 @@ _exit:
return code; return code;
} }
static int32_t tsdbCommitDelData(SCommitter2 *pCommitter) { static int32_t tsdbCommitDelData(SCommitter2 *committer) {
int32_t code = 0; int32_t code = 0;
int32_t lino; int32_t lino;
@ -226,23 +233,23 @@ static int32_t tsdbCommitDelData(SCommitter2 *pCommitter) {
ASSERTS(0, "TODO: Not implemented yet"); ASSERTS(0, "TODO: Not implemented yet");
int64_t nDel = 0; int64_t nDel = 0;
SMemTable *pMem = pCommitter->tsdb->imem; SMemTable *pMem = committer->tsdb->imem;
if (pMem->nDel == 0) { // no del data if (pMem->nDel == 0) { // no del data
goto _exit; goto _exit;
} }
for (int32_t iTbData = 0; iTbData < taosArrayGetSize(pCommitter->aTbDataP); iTbData++) { for (int32_t iTbData = 0; iTbData < taosArrayGetSize(committer->aTbDataP); iTbData++) {
STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData); STbData *pTbData = (STbData *)taosArrayGetP(committer->aTbDataP, iTbData);
for (SDelData *pDelData = pTbData->pHead; pDelData; pDelData = pDelData->pNext) { for (SDelData *pDelData = pTbData->pHead; pDelData; pDelData = pDelData->pNext) {
if (pDelData->eKey < pCommitter->ctx->minKey) continue; if (pDelData->eKey < committer->ctx->minKey) continue;
if (pDelData->sKey > pCommitter->ctx->maxKey) { if (pDelData->sKey > committer->ctx->maxKey) {
pCommitter->ctx->nextKey = TMIN(pCommitter->ctx->nextKey, pDelData->sKey); committer->ctx->nextKey = TMIN(committer->ctx->nextKey, pDelData->sKey);
continue; continue;
} }
code = tsdbCommitWriteDelData(pCommitter, pTbData->suid, pTbData->uid, pDelData->version, code = tsdbCommitWriteDelData(committer, pTbData->suid, pTbData->uid, pDelData->version,
pDelData->sKey /* TODO */, pDelData->eKey /* TODO */); pDelData->sKey /* TODO */, pDelData->eKey /* TODO */);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
@ -250,9 +257,9 @@ static int32_t tsdbCommitDelData(SCommitter2 *pCommitter) {
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d failed at line %d since %s", TD_VID(pCommitter->tsdb->pVnode), lino, tstrerror(code)); tsdbError("vgId:%d failed at line %d since %s", TD_VID(committer->tsdb->pVnode), lino, tstrerror(code));
} else { } else {
tsdbDebug("vgId:%d %s done, fid:%d nDel:%" PRId64, TD_VID(pCommitter->tsdb->pVnode), __func__, pCommitter->ctx->fid, tsdbDebug("vgId:%d %s done, fid:%d nDel:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->fid,
pMem->nDel); pMem->nDel);
} }
return code; return code;
@ -266,17 +273,26 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) {
int32_t vid = TD_VID(tsdb->pVnode); int32_t vid = TD_VID(tsdb->pVnode);
committer->ctx->fid = tsdbKeyFid(committer->ctx->nextKey, committer->minutes, committer->precision); committer->ctx->fid = tsdbKeyFid(committer->ctx->nextKey, committer->minutes, committer->precision);
committer->ctx->expLevel = tsdbFidLevel(committer->ctx->fid, &tsdb->keepCfg, committer->ctx->now);
tsdbFidKeyRange(committer->ctx->fid, committer->minutes, committer->precision, &committer->ctx->minKey, tsdbFidKeyRange(committer->ctx->fid, committer->minutes, committer->precision, &committer->ctx->minKey,
&committer->ctx->maxKey); &committer->ctx->maxKey);
committer->ctx->expLevel = tsdbFidLevel(committer->ctx->fid, &tsdb->keepCfg, committer->ctx->now); STFileSet fset = {.fid = committer->ctx->fid};
committer->ctx->nextKey = TSKEY_MAX; committer->ctx->fset = &fset;
committer->ctx->fset = TARRAY2_SEARCH_EX(committer->fsetArr, &committer->ctx->fset, tsdbTFileSetCmprFn, TD_EQ);
committer->ctx->tbid->suid = 0;
committer->ctx->tbid->uid = 0;
// TODO: use a thread safe function to get fset ASSERT(TARRAY2_SIZE(committer->iterArray) == 0);
tsdbFSGetFSet(tsdb->pFS, committer->ctx->fid, &committer->ctx->fset); ASSERT(committer->iterMerger == NULL);
ASSERT(committer->sttWriter == NULL);
ASSERT(committer->dataWriter == NULL);
code = tsdbCommitOpenWriter(committer); code = tsdbCommitOpenWriter(committer);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// reset nextKey
committer->ctx->nextKey = TSKEY_MAX;
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(vid, lino, code); TSDB_ERROR_LOG(vid, lino, code);
@ -287,27 +303,27 @@ _exit:
return 0; return 0;
} }
static int32_t tsdbCommitFileSetEnd(SCommitter2 *pCommitter) { static int32_t tsdbCommitFileSetEnd(SCommitter2 *committer) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
int32_t vid = TD_VID(pCommitter->tsdb->pVnode);
if (pCommitter->sttWriter == NULL) return 0; STFileOp op[1];
code = tsdbSttFileWriterClose(&committer->sttWriter, 0, op);
STFileOp op;
code = tsdbSttFileWriterClose(&pCommitter->sttWriter, 0, &op);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
if (op.optype != TSDB_FOP_NONE) { if (op->optype != TSDB_FOP_NONE) {
code = TARRAY2_APPEND(pCommitter->fopArray, op); code = TARRAY2_APPEND_PTR(committer->fopArray, op);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
tsdbIterMergerClose(&committer->iterMerger);
TARRAY2_CLEAR(committer->iterArray, tsdbIterClose);
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d failed at line %d since %s", vid, lino, tstrerror(code)); TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code);
} else { } else {
tsdbDebug("vgId:%d %s done, fid:%d", vid, __func__, pCommitter->ctx->fid); tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->fid);
} }
return code; return code;
} }
@ -349,20 +365,24 @@ static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *co
memset(committer, 0, sizeof(committer[0])); memset(committer, 0, sizeof(committer[0]));
committer->tsdb = tsdb; committer->tsdb = tsdb;
code = tsdbFSCreateCopySnapshot(tsdb->pFS, &committer->fsetArr);
TSDB_CHECK_CODE(code, lino, _exit);
committer->minutes = tsdb->keepCfg.days; committer->minutes = tsdb->keepCfg.days;
committer->precision = tsdb->keepCfg.precision; committer->precision = tsdb->keepCfg.precision;
committer->minRow = info->info.config.tsdbCfg.minRows; committer->minRow = info->info.config.tsdbCfg.minRows;
committer->maxRow = info->info.config.tsdbCfg.maxRows; committer->maxRow = info->info.config.tsdbCfg.maxRows;
committer->cmprAlg = info->info.config.tsdbCfg.compression; committer->cmprAlg = info->info.config.tsdbCfg.compression;
committer->sttTrigger = info->info.config.sttTrigger; committer->sttTrigger = info->info.config.sttTrigger;
committer->compactVersion = INT64_MAX; // TODO: use a function committer->szPage = info->info.config.tsdbPageSize;
committer->compactVersion = INT64_MAX;
TARRAY2_INIT(committer->fopArray);
tsdbFSAllocEid(tsdb->pFS, &committer->eid);
committer->ctx->cid = tsdbFSAllocEid(tsdb->pFS);
committer->ctx->now = taosGetTimestampSec(); committer->ctx->now = taosGetTimestampSec();
committer->ctx->nextKey = tsdb->imem->minKey; // TODO committer->ctx->nextKey = tsdb->imem->minKey; // TODO
TARRAY2_INIT(committer->fopArray);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(vid, lino, code); TSDB_ERROR_LOG(vid, lino, code);
@ -372,28 +392,32 @@ _exit:
return code; return code;
} }
static int32_t tsdbCloseCommitter(SCommitter2 *pCommiter, int32_t eno) { static int32_t tsdbCloseCommitter(SCommitter2 *committer, int32_t eno) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
int32_t vid = TD_VID(pCommiter->tsdb->pVnode); int32_t vid = TD_VID(committer->tsdb->pVnode);
if (eno == 0) { if (eno == 0) {
code = tsdbFSEditBegin(pCommiter->tsdb->pFS, pCommiter->fopArray, TSDB_FEDIT_COMMIT); code = tsdbFSEditBegin(committer->tsdb->pFS, committer->fopArray, TSDB_FEDIT_COMMIT);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
// TODO // TODO
ASSERT(0); ASSERT(0);
} }
ASSERT(pCommiter->sttWriter == NULL); ASSERT(committer->dataWriter == NULL);
TARRAY2_FREE(pCommiter->fopArray); ASSERT(committer->sttWriter == NULL);
ASSERT(committer->iterMerger == NULL);
TARRAY2_FREE(committer->iterArray);
TARRAY2_FREE(committer->fopArray);
tsdbFSDestroyCopySnapshot(&committer->fsetArr);
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed at line %d since %s, eid:%" PRId64, vid, __func__, lino, tstrerror(code), tsdbError("vgId:%d %s failed at line %d since %s, eid:%" PRId64, vid, __func__, lino, tstrerror(code),
pCommiter->eid); committer->ctx->cid);
} else { } else {
tsdbDebug("vgId:%d %s done, eid:%" PRId64, vid, __func__, pCommiter->eid); tsdbDebug("vgId:%d %s done, eid:%" PRId64, vid, __func__, committer->ctx->cid);
} }
return code; return code;
} }
@ -413,15 +437,15 @@ int32_t tsdbCommitBegin(STsdb *tsdb, SCommitInfo *info) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
int32_t vid = TD_VID(tsdb->pVnode); int32_t vid = TD_VID(tsdb->pVnode);
SMemTable *memt = tsdb->imem; SMemTable *imem = tsdb->imem;
int64_t nRow = memt->nRow; int64_t nRow = imem->nRow;
int64_t nDel = memt->nDel; int64_t nDel = imem->nDel;
if (!nRow && !nDel) { if (!nRow && !nDel) {
taosThreadRwlockWrlock(&tsdb->rwLock); taosThreadRwlockWrlock(&tsdb->rwLock);
tsdb->imem = NULL; tsdb->imem = NULL;
taosThreadRwlockUnlock(&tsdb->rwLock); taosThreadRwlockUnlock(&tsdb->rwLock);
tsdbUnrefMemTable(memt, NULL, true); tsdbUnrefMemTable(imem, NULL, true);
} else { } else {
SCommitter2 committer[1]; SCommitter2 committer[1];
@ -430,10 +454,7 @@ int32_t tsdbCommitBegin(STsdb *tsdb, SCommitInfo *info) {
while (committer->ctx->nextKey != TSKEY_MAX) { while (committer->ctx->nextKey != TSKEY_MAX) {
code = tsdbCommitFileSet(committer); code = tsdbCommitFileSet(committer);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
lino = __LINE__;
break;
}
} }
code = tsdbCloseCommitter(committer, code); code = tsdbCloseCommitter(committer, code);
@ -449,33 +470,32 @@ _exit:
return code; return code;
} }
int32_t tsdbCommitCommit(STsdb *pTsdb) { int32_t tsdbCommitCommit(STsdb *tsdb) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
int32_t vid = TD_VID(pTsdb->pVnode);
if (pTsdb->imem == NULL) goto _exit; if (tsdb->imem == NULL) goto _exit;
SMemTable *pMemTable = pTsdb->imem; SMemTable *pMemTable = tsdb->imem;
taosThreadRwlockWrlock(&pTsdb->rwLock); taosThreadRwlockWrlock(&tsdb->rwLock);
code = tsdbFSEditCommit(pTsdb->pFS); code = tsdbFSEditCommit(tsdb->pFS);
if (code) { if (code) {
taosThreadRwlockUnlock(&pTsdb->rwLock); taosThreadRwlockUnlock(&tsdb->rwLock);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
pTsdb->imem = NULL; tsdb->imem = NULL;
taosThreadRwlockUnlock(&pTsdb->rwLock); taosThreadRwlockUnlock(&tsdb->rwLock);
tsdbUnrefMemTable(pMemTable, NULL, true); tsdbUnrefMemTable(pMemTable, NULL, true);
// TODO: make this call async // TODO: make this call async
code = tsdbMerge(pTsdb); code = tsdbMerge(tsdb);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d, %s failed at line %d since %s", vid, __func__, lino, tstrerror(code)); TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
} else { } else {
tsdbInfo("vgId:%d %s done", vid, __func__); tsdbInfo("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__);
} }
return code; return code;
} }

View File

@ -538,9 +538,11 @@ int32_t tsdbCloseFS(STFileSystem **ppFS) {
return 0; return 0;
} }
int32_t tsdbFSAllocEid(STFileSystem *pFS, int64_t *eid) { int64_t tsdbFSAllocEid(STFileSystem *fs) {
eid[0] = ++pFS->neid; // TODO: use atomic operation taosThreadRwlockRdlock(&fs->tsdb->rwLock);
return 0; int64_t cid = ++fs->neid;
taosThreadRwlockUnlock(&fs->tsdb->rwLock);
return cid;
} }
int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype) { int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype) {

View File

@ -267,7 +267,7 @@ int32_t tsdbTFileSetEdit(STsdb *pTsdb, STFileSet *fset, const STFileOp *op) {
if (code) return code; if (code) return code;
if (fobj->f->type == TSDB_FTYPE_STT) { if (fobj->f->type == TSDB_FTYPE_STT) {
SSttLvl *lvl = tsdbTFileSetGetLvl(fset, fobj->f->stt->level); SSttLvl *lvl = tsdbTFileSetGetSttLvl(fset, fobj->f->stt->level);
if (!lvl) { if (!lvl) {
code = tsdbSttLvlInit(fobj->f->stt->level, &lvl); code = tsdbSttLvlInit(fobj->f->stt->level, &lvl);
if (code) return code; if (code) return code;
@ -285,7 +285,7 @@ int32_t tsdbTFileSetEdit(STsdb *pTsdb, STFileSet *fset, const STFileOp *op) {
} else if (op->optype == TSDB_FOP_REMOVE) { } else if (op->optype == TSDB_FOP_REMOVE) {
// delete a file // delete a file
if (op->of.type == TSDB_FTYPE_STT) { if (op->of.type == TSDB_FTYPE_STT) {
SSttLvl *lvl = tsdbTFileSetGetLvl(fset, op->of.stt->level); SSttLvl *lvl = tsdbTFileSetGetSttLvl(fset, op->of.stt->level);
ASSERT(lvl); ASSERT(lvl);
STFileObj tfobj = {.f[0] = {.cid = op->of.cid}}; STFileObj tfobj = {.f[0] = {.cid = op->of.cid}};
@ -305,7 +305,7 @@ int32_t tsdbTFileSetEdit(STsdb *pTsdb, STFileSet *fset, const STFileOp *op) {
} }
} else { } else {
if (op->nf.type == TSDB_FTYPE_STT) { if (op->nf.type == TSDB_FTYPE_STT) {
SSttLvl *lvl = tsdbTFileSetGetLvl(fset, op->of.stt->level); SSttLvl *lvl = tsdbTFileSetGetSttLvl(fset, op->of.stt->level);
ASSERT(lvl); ASSERT(lvl);
STFileObj tfobj = {.f[0] = {.cid = op->of.cid}}, *tfobjp = &tfobj; STFileObj tfobj = {.f[0] = {.cid = op->of.cid}}, *tfobjp = &tfobj;
@ -463,9 +463,9 @@ int32_t tsdbTFileSetRemove(STFileSet **fset) {
return 0; return 0;
} }
SSttLvl *tsdbTFileSetGetLvl(STFileSet *fset, int32_t level) { SSttLvl *tsdbTFileSetGetSttLvl(STFileSet *fset, int32_t level) {
SSttLvl tlvl = {.level = level}; SSttLvl sttLvl = {.level = level};
SSttLvl *lvl = &tlvl; SSttLvl *lvl = &sttLvl;
return TARRAY2_SEARCH_EX(fset->lvlArr, &lvl, tsdbSttLvlCmprFn, TD_EQ); return TARRAY2_SEARCH_EX(fset->lvlArr, &lvl, tsdbSttLvlCmprFn, TD_EQ);
} }

View File

@ -348,7 +348,7 @@ struct SIterMerger {
SRBTree iterTree[1]; SRBTree iterTree[1];
}; };
int32_t tsdbIterMergerInit(const TTsdbIterArray *iterArray, SIterMerger **merger) { int32_t tsdbIterMergerOpen(const TTsdbIterArray *iterArray, SIterMerger **merger) {
STsdbIter *iter; STsdbIter *iter;
SRBTreeNode *node; SRBTreeNode *node;
@ -365,7 +365,7 @@ int32_t tsdbIterMergerInit(const TTsdbIterArray *iterArray, SIterMerger **merger
return tsdbIterMergerNext(merger[0]); return tsdbIterMergerNext(merger[0]);
} }
int32_t tsdbIterMergerClear(SIterMerger **merger) { int32_t tsdbIterMergerClose(SIterMerger **merger) {
if (merger[0]) { if (merger[0]) {
taosMemoryFree(merger[0]); taosMemoryFree(merger[0]);
merger[0] = NULL; merger[0] = NULL;

View File

@ -57,7 +57,7 @@ static int32_t tsdbMergerOpen(SMerger *merger) {
merger->szPage = merger->tsdb->pVnode->config.tsdbPageSize; merger->szPage = merger->tsdb->pVnode->config.tsdbPageSize;
merger->cmprAlg = merger->tsdb->pVnode->config.tsdbCfg.compression; merger->cmprAlg = merger->tsdb->pVnode->config.tsdbCfg.compression;
merger->compactVersion = INT64_MAX; merger->compactVersion = INT64_MAX;
tsdbFSAllocEid(merger->tsdb->pFS, &merger->cid); merger->cid = tsdbFSAllocEid(merger->tsdb->pFS);
merger->ctx->opened = true; merger->ctx->opened = true;
return 0; return 0;
} }
@ -273,7 +273,7 @@ static int32_t tsdbMergeFileSetBeginOpenIter(SMerger *merger) {
} }
} }
code = tsdbIterMergerInit(merger->iterArr, &merger->iterMerger); code = tsdbIterMergerOpen(merger->iterArr, &merger->iterMerger);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
@ -444,7 +444,7 @@ _exit:
} }
static int32_t tsdbMergeFileSetEndCloseIter(SMerger *merger) { static int32_t tsdbMergeFileSetEndCloseIter(SMerger *merger) {
tsdbIterMergerClear(&merger->iterMerger); tsdbIterMergerClose(&merger->iterMerger);
TARRAY2_CLEAR(merger->iterArr, tsdbIterClose); TARRAY2_CLEAR(merger->iterArr, tsdbIterClose);
return 0; return 0;
} }
@ -566,5 +566,5 @@ _exit:
} else if (merger->ctx->opened) { } else if (merger->ctx->opened) {
tsdbDebug("vgId:%d %s done", vid, __func__); tsdbDebug("vgId:%d %s done", vid, __func__);
} }
return 0; return code;
} }

View File

@ -28,6 +28,7 @@ struct SSttFileReader {
SSttFileReaderConfig config[1]; SSttFileReaderConfig config[1];
TSttSegReaderArray readerArray[1]; TSttSegReaderArray readerArray[1];
STsdbFD *fd; STsdbFD *fd;
uint8_t *bufArr[5];
}; };
struct SSttSegReader { struct SSttSegReader {
@ -87,6 +88,7 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con
if (reader[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; if (reader[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY;
reader[0]->config[0] = config[0]; reader[0]->config[0] = config[0];
if (!reader[0]->config->bufArr) reader[0]->config->bufArr = reader[0]->bufArr;
// open file // open file
code = tsdbOpenFile(fname, config->szPage, TD_FILE_READ, &reader[0]->fd); code = tsdbOpenFile(fname, config->szPage, TD_FILE_READ, &reader[0]->fd);
@ -118,6 +120,9 @@ _exit:
int32_t tsdbSttFileReaderClose(SSttFileReader **reader) { int32_t tsdbSttFileReaderClose(SSttFileReader **reader) {
if (reader[0]) { if (reader[0]) {
for (int32_t i = 0; i < ARRAY_SIZE(reader[0]->bufArr); ++i) {
tFree(reader[0]->bufArr[i]);
}
tsdbCloseFile(&reader[0]->fd); tsdbCloseFile(&reader[0]->fd);
TARRAY2_CLEAR_FREE(reader[0]->readerArray, tsdbSttSegReaderClose); TARRAY2_CLEAR_FREE(reader[0]->readerArray, tsdbSttSegReaderClose);
taosMemoryFree(reader[0]); taosMemoryFree(reader[0]);