xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refact more code

This commit is contained in:
Hongze Cheng 2024-07-17 22:09:09 +08:00
parent a3c0acb78a
commit decd68f811
14 changed files with 872 additions and 1204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
source/dnode/vnode/src/inc/tsdb.h
View File

@ -259,7 +259,6 @@ int32_t tsdbGetNRowsInTbData(STbData *pTbData);
typedef enum { TSDB_HEAD_FILE = 0, TSDB_DATA_FILE, TSDB_LAST_FILE, TSDB_SMA_FILE } EDataFileT; typedef enum { TSDB_HEAD_FILE = 0, TSDB_DATA_FILE, TSDB_LAST_FILE, TSDB_SMA_FILE } EDataFileT;
bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2); bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2);
int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype);
int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile); int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile);
int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile); int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile);
int32_t tPutSttFile(uint8_t *p, SSttFile *pSttFile); int32_t tPutSttFile(uint8_t *p, SSttFile *pSttFile);

4
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -222,8 +222,8 @@ typedef struct SMetaInfo {
int32_t metaGetInfo(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo, SMetaReader* pReader); int32_t metaGetInfo(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo, SMetaReader* pReader);
// tsdb // tsdb
int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback, bool force); int32_t tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback, bool force);
int tsdbClose(STsdb** pTsdb); int32_t tsdbClose(STsdb** pTsdb);
int32_t tsdbBegin(STsdb* pTsdb); int32_t tsdbBegin(STsdb* pTsdb);
// int32_t tsdbPrepareCommit(STsdb* pTsdb); // int32_t tsdbPrepareCommit(STsdb* pTsdb);
// int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo); // int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo);

11
source/dnode/vnode/src/tsdb/tsdbCommit.c
View File

@ -41,26 +41,23 @@ int32_t tsdbBegin(STsdb *pTsdb) {
if (!pTsdb) return code; if (!pTsdb) return code;
SMemTable *pMemTable; SMemTable *pMemTable;
code = tsdbMemTableCreate(pTsdb, &pMemTable); TAOS_CHECK_GOTO(tsdbMemTableCreate(pTsdb, &pMemTable), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// lock // lock
if ((code = taosThreadMutexLock(&pTsdb->mutex))) { if ((code = taosThreadMutexLock(&pTsdb->mutex))) {
code = TAOS_SYSTEM_ERROR(code); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(code), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
pTsdb->mem = pMemTable; pTsdb->mem = pMemTable;
// unlock // unlock
if ((code = taosThreadMutexUnlock(&pTsdb->mutex))) { if ((code = taosThreadMutexUnlock(&pTsdb->mutex))) {
code = TAOS_SYSTEM_ERROR(code); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(code), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pTsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }

205
source/dnode/vnode/src/tsdb/tsdbCommit2.c
View File

@ -92,12 +92,12 @@ static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) {
} }
} }
code = tsdbFSetWriterOpen(&config, &committer->writer); TAOS_CHECK_GOTO(tsdbFSetWriterOpen(&config, &committer->writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -122,32 +122,28 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) {
committer->ctx->tbid->uid = row->uid; committer->ctx->tbid->uid = row->uid;
if (metaGetInfo(committer->tsdb->pVnode->pMeta, row->uid, &info, NULL) != 0) { if (metaGetInfo(committer->tsdb->pVnode->pMeta, row->uid, &info, NULL) != 0) {
code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid); TAOS_CHECK_GOTO(tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
continue; continue;
} }
} }
int64_t ts = TSDBROW_TS(&row->row); int64_t ts = TSDBROW_TS(&row->row);
if (ts > committer->ctx->maxKey) { if (ts > committer->ctx->maxKey) {
code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid); TAOS_CHECK_GOTO(tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
continue; continue;
} }
committer->ctx->hasTSData = true; committer->ctx->hasTSData = true;
numOfRow++; numOfRow++;
code = tsdbFSetWriteRow(committer->writer, row); TAOS_CHECK_GOTO(tsdbFSetWriteRow(committer->writer, row), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tsdbIterMergerNext(committer->dataIterMerger), &lino, _exit);
code = tsdbIterMergerNext(committer->dataIterMerger);
TSDB_CHECK_CODE(code, lino, _exit);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} else { } else {
tsdbDebug("vgId:%d fid:%d commit %" PRId64 " rows", TD_VID(committer->tsdb->pVnode), committer->ctx->info->fid, tsdbDebug("vgId:%d fid:%d commit %" PRId64 " rows", TD_VID(committer->tsdb->pVnode), committer->ctx->info->fid,
numOfRow); numOfRow);
@ -176,8 +172,7 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) {
committer->ctx->tbid->uid = record->uid; committer->ctx->tbid->uid = record->uid;
if (metaGetInfo(committer->tsdb->pVnode->pMeta, record->uid, &info, NULL) != 0) { if (metaGetInfo(committer->tsdb->pVnode->pMeta, record->uid, &info, NULL) != 0) {
code = tsdbIterMergerSkipTableData(committer->tombIterMerger, committer->ctx->tbid); TAOS_CHECK_GOTO(tsdbIterMergerSkipTableData(committer->tombIterMerger, committer->ctx->tbid), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
continue; continue;
} }
} }
@ -192,18 +187,17 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) {
if (!skip) { if (!skip) {
numRecord++; numRecord++;
code = tsdbFSetWriteTombRecord(committer->writer, record); TAOS_CHECK_GOTO(tsdbFSetWriteTombRecord(committer->writer, record), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
code = tsdbIterMergerNext(committer->tombIterMerger); TAOS_CHECK_GOTO(tsdbIterMergerNext(committer->tombIterMerger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} else { } else {
tsdbDebug("vgId:%d fid:%d commit %" PRId64 " tomb records", TD_VID(committer->tsdb->pVnode), tsdbDebug("vgId:%d fid:%d commit %" PRId64 " tomb records", TD_VID(committer->tsdb->pVnode),
committer->ctx->info->fid, numRecord); committer->ctx->info->fid, numRecord);
@ -241,11 +235,9 @@ static int32_t tsdbCommitOpenReader(SCommitter2 *committer) {
.file = fobj->f[0], .file = fobj->f[0],
}; };
code = tsdbSttFileReaderOpen(fobj->fname, &config, &sttReader); TAOS_CHECK_GOTO(tsdbSttFileReaderOpen(fobj->fname, &config, &sttReader), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = TARRAY2_APPEND(committer->sttReaderArray, sttReader); TAOS_CHECK_GOTO(TARRAY2_APPEND(committer->sttReaderArray, sttReader), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
STFileOp op = { STFileOp op = {
.optype = TSDB_FOP_REMOVE, .optype = TSDB_FOP_REMOVE,
@ -253,15 +245,15 @@ static int32_t tsdbCommitOpenReader(SCommitter2 *committer) {
.of = fobj->f[0], .of = fobj->f[0],
}; };
code = TARRAY2_APPEND(committer->fopArray, op); TAOS_CHECK_GOTO(TARRAY2_APPEND(committer->fopArray, op), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
_exit: _exit:
if (code) { if (code) {
tsdbCommitCloseReader(committer); tsdbCommitCloseReader(committer);
TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -295,21 +287,15 @@ static int32_t tsdbCommitOpenIter(SCommitter2 *committer) {
.numOfPKs = 0, .numOfPKs = 0,
}; };
code = tsdbIterOpen(&config, &iter); TAOS_CHECK_GOTO(tsdbIterOpen(&config, &iter), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(TARRAY2_APPEND(committer->dataIterArray, iter), &lino, _exit);
code = TARRAY2_APPEND(committer->dataIterArray, iter);
TSDB_CHECK_CODE(code, lino, _exit);
// mem tomb iter // mem tomb iter
config.type = TSDB_ITER_TYPE_MEMT_TOMB; config.type = TSDB_ITER_TYPE_MEMT_TOMB;
config.memt = committer->tsdb->imem; config.memt = committer->tsdb->imem;
code = tsdbIterOpen(&config, &iter); TAOS_CHECK_GOTO(tsdbIterOpen(&config, &iter), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(TARRAY2_APPEND(committer->tombIterArray, iter), &lino, _exit);
code = TARRAY2_APPEND(committer->tombIterArray, iter);
TSDB_CHECK_CODE(code, lino, _exit);
// STT // STT
SSttFileReader *sttReader; SSttFileReader *sttReader;
@ -318,34 +304,27 @@ static int32_t tsdbCommitOpenIter(SCommitter2 *committer) {
config.type = TSDB_ITER_TYPE_STT; config.type = TSDB_ITER_TYPE_STT;
config.sttReader = sttReader; config.sttReader = sttReader;
code = tsdbIterOpen(&config, &iter); TAOS_CHECK_GOTO(tsdbIterOpen(&config, &iter), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(TARRAY2_APPEND(committer->dataIterArray, iter), &lino, _exit);
code = TARRAY2_APPEND(committer->dataIterArray, iter);
TSDB_CHECK_CODE(code, lino, _exit);
// tomb iter // tomb iter
config.type = TSDB_ITER_TYPE_STT_TOMB; config.type = TSDB_ITER_TYPE_STT_TOMB;
config.sttReader = sttReader; config.sttReader = sttReader;
code = tsdbIterOpen(&config, &iter); TAOS_CHECK_GOTO(tsdbIterOpen(&config, &iter), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = TARRAY2_APPEND(committer->tombIterArray, iter); TAOS_CHECK_GOTO(TARRAY2_APPEND(committer->tombIterArray, iter), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
// open merger // open merger
code = tsdbIterMergerOpen(committer->dataIterArray, &committer->dataIterMerger, false); TAOS_CHECK_GOTO(tsdbIterMergerOpen(committer->dataIterArray, &committer->dataIterMerger, false), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tsdbIterMergerOpen(committer->tombIterArray, &committer->tombIterMerger, true), &lino, _exit);
code = tsdbIterMergerOpen(committer->tombIterArray, &committer->tombIterMerger, true);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
tsdbCommitCloseIter(committer); tsdbCommitCloseIter(committer);
TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -361,8 +340,10 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) {
committer->ctx->expLevel = tsdbFidLevel(committer->ctx->info->fid, &tsdb->keepCfg, committer->now); committer->ctx->expLevel = tsdbFidLevel(committer->ctx->info->fid, &tsdb->keepCfg, committer->now);
tsdbFidKeyRange(committer->ctx->info->fid, committer->minutes, committer->precision, &committer->ctx->minKey, tsdbFidKeyRange(committer->ctx->info->fid, committer->minutes, committer->precision, &committer->ctx->minKey,
&committer->ctx->maxKey); &committer->ctx->maxKey);
code = tfsAllocDisk(committer->tsdb->pVnode->pTfs, committer->ctx->expLevel, &committer->ctx->did);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tfsAllocDisk(committer->tsdb->pVnode->pTfs, committer->ctx->expLevel, &committer->ctx->did), &lino,
_exit);
tfsMkdirRecurAt(committer->tsdb->pVnode->pTfs, committer->tsdb->path, committer->ctx->did); tfsMkdirRecurAt(committer->tsdb->pVnode->pTfs, committer->tsdb->path, committer->ctx->did);
committer->ctx->tbid->suid = 0; committer->ctx->tbid->suid = 0;
committer->ctx->tbid->uid = 0; committer->ctx->tbid->uid = 0;
@ -371,18 +352,13 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) {
ASSERT(committer->dataIterMerger == NULL); ASSERT(committer->dataIterMerger == NULL);
ASSERT(committer->writer == NULL); ASSERT(committer->writer == NULL);
code = tsdbCommitOpenReader(committer); TAOS_CHECK_GOTO(tsdbCommitOpenReader(committer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tsdbCommitOpenIter(committer), &lino, _exit);
TAOS_CHECK_GOTO(tsdbCommitOpenWriter(committer), &lino, _exit);
code = tsdbCommitOpenIter(committer);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbCommitOpenWriter(committer);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} else { } else {
tsdbDebug("vgId:%d %s done, fid:%d minKey:%" PRId64 " maxKey:%" PRId64 " expLevel:%d", TD_VID(tsdb->pVnode), tsdbDebug("vgId:%d %s done, fid:%d minKey:%" PRId64 " maxKey:%" PRId64 " expLevel:%d", TD_VID(tsdb->pVnode),
__func__, committer->ctx->info->fid, committer->ctx->minKey, committer->ctx->maxKey, __func__, committer->ctx->info->fid, committer->ctx->minKey, committer->ctx->maxKey,
@ -395,18 +371,14 @@ static int32_t tsdbCommitFileSetEnd(SCommitter2 *committer) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
code = tsdbCommitCloseWriter(committer); TAOS_CHECK_GOTO(tsdbCommitCloseWriter(committer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tsdbCommitCloseIter(committer), &lino, _exit);
TAOS_CHECK_GOTO(tsdbCommitCloseReader(committer), &lino, _exit);
code = tsdbCommitCloseIter(committer);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbCommitCloseReader(committer);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} else { } else {
tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->info->fid); tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->info->fid);
} }
@ -417,24 +389,15 @@ static int32_t tsdbCommitFileSet(SCommitter2 *committer) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
// fset commit start TAOS_CHECK_GOTO(tsdbCommitFileSetBegin(committer), &lino, _exit);
code = tsdbCommitFileSetBegin(committer); TAOS_CHECK_GOTO(tsdbCommitTSData(committer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tsdbCommitTombData(committer), &lino, _exit);
TAOS_CHECK_GOTO(tsdbCommitFileSetEnd(committer), &lino, _exit);
// commit fset
code = tsdbCommitTSData(committer);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbCommitTombData(committer);
TSDB_CHECK_CODE(code, lino, _exit);
// fset commit end
code = tsdbCommitFileSetEnd(committer);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} else { } else {
tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->info->fid); tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->info->fid);
} }
@ -464,9 +427,6 @@ static uint32_t tFileSetCommitInfoHash(const void *arg) {
} }
static int32_t tsdbCommitInfoDestroy(STsdb *pTsdb) { static int32_t tsdbCommitInfoDestroy(STsdb *pTsdb) {
int32_t code = 0;
int32_t lino = 0;
if (pTsdb->commitInfo) { if (pTsdb->commitInfo) {
for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) { for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) {
SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(pTsdb->commitInfo->arr, i); SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(pTsdb->commitInfo->arr, i);
@ -480,12 +440,7 @@ static int32_t tsdbCommitInfoDestroy(STsdb *pTsdb) {
pTsdb->commitInfo->arr = NULL; pTsdb->commitInfo->arr = NULL;
taosMemoryFreeClear(pTsdb->commitInfo); taosMemoryFreeClear(pTsdb->commitInfo);
} }
return 0;
_exit:
if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
}
return code;
} }
static int32_t tsdbCommitInfoInit(STsdb *pTsdb) { static int32_t tsdbCommitInfoInit(STsdb *pTsdb) {
@ -494,11 +449,10 @@ static int32_t tsdbCommitInfoInit(STsdb *pTsdb) {
pTsdb->commitInfo = taosMemoryCalloc(1, sizeof(*pTsdb->commitInfo)); pTsdb->commitInfo = taosMemoryCalloc(1, sizeof(*pTsdb->commitInfo));
if (pTsdb->commitInfo == NULL) { if (pTsdb->commitInfo == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
} }
code = vHashInit(&pTsdb->commitInfo->ht, tFileSetCommitInfoHash, tFileSetCommitInfoCompare); TAOS_CHECK_GOTO(vHashInit(&pTsdb->commitInfo->ht, tFileSetCommitInfoHash, tFileSetCommitInfoCompare), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
pTsdb->commitInfo->arr = taosArrayInit(0, sizeof(SFileSetCommitInfo *)); pTsdb->commitInfo->arr = taosArrayInit(0, sizeof(SFileSetCommitInfo *));
if (pTsdb->commitInfo->arr == NULL) { if (pTsdb->commitInfo->arr == NULL) {
@ -508,7 +462,7 @@ static int32_t tsdbCommitInfoInit(STsdb *pTsdb) {
_exit: _exit:
if (code) { if (code) {
tsdbCommitInfoDestroy(pTsdb); tsdbCommitInfoDestroy(pTsdb);
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(pTsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -520,22 +474,21 @@ static int32_t tsdbCommitInfoAdd(STsdb *tsdb, int32_t fid) {
SFileSetCommitInfo *tinfo; SFileSetCommitInfo *tinfo;
if ((tinfo = taosMemoryMalloc(sizeof(*tinfo))) == NULL) { if ((tinfo = taosMemoryMalloc(sizeof(*tinfo))) == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
} }
tinfo->fid = fid; tinfo->fid = fid;
tinfo->fset = NULL; tinfo->fset = NULL;
code = vHashPut(tsdb->commitInfo->ht, tinfo); TAOS_CHECK_GOTO(vHashPut(tsdb->commitInfo->ht, tinfo), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
if ((taosArrayPush(tsdb->commitInfo->arr, &tinfo)) == NULL) { if ((taosArrayPush(tsdb->commitInfo->arr, &tinfo)) == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
} }
taosArraySort(tsdb->commitInfo->arr, tFileSetCommitInfoPCompare); taosArraySort(tsdb->commitInfo->arr, tFileSetCommitInfoPCompare);
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code)); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -547,8 +500,7 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
STFileSet *fset = NULL; STFileSet *fset = NULL;
SRBTreeIter iter; SRBTreeIter iter;
code = tsdbCommitInfoInit(tsdb); TAOS_CHECK_GOTO(tsdbCommitInfoInit(tsdb), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// scan time-series data // scan time-series data
iter = tRBTreeIterCreate(tsdb->imem->tbDataTree, 1); iter = tRBTreeIterCreate(tsdb->imem->tbDataTree, 1);
@ -581,8 +533,7 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
}; };
vHashGet(tsdb->commitInfo->ht, &tinfo, (void **)&info); vHashGet(tsdb->commitInfo->ht, &tinfo, (void **)&info);
if (info == NULL) { if (info == NULL) {
code = tsdbCommitInfoAdd(tsdb, fid); TAOS_CHECK_GOTO(tsdbCommitInfoAdd(tsdb, fid), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
from.key.ts = maxKey + 1; from.key.ts = maxKey + 1;
@ -640,7 +591,11 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i); SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i);
tsdbBeginTaskOnFileSet(tsdb, info->fid, &fset); tsdbBeginTaskOnFileSet(tsdb, info->fid, &fset);
if (fset) { if (fset) {
tsdbTFileSetInitCopy(tsdb, fset, &info->fset); code = tsdbTFileSetInitCopy(tsdb, fset, &info->fset);
if (code) {
taosThreadMutexUnlock(&tsdb->mutex);
TAOS_CHECK_GOTO(code, &lino, _exit);
}
} }
} }
@ -649,7 +604,7 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
_exit: _exit:
if (code) { if (code) {
tsdbCommitInfoDestroy(tsdb); tsdbCommitInfoDestroy(tsdb);
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code)); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -670,12 +625,11 @@ static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *co
committer->cid = tsdbFSAllocEid(tsdb->pFS); committer->cid = tsdbFSAllocEid(tsdb->pFS);
committer->now = taosGetTimestampSec(); committer->now = taosGetTimestampSec();
code = tsdbCommitInfoBuild(tsdb); TAOS_CHECK_GOTO(tsdbCommitInfoBuild(tsdb), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} else { } else {
tsdbDebug("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); tsdbDebug("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__);
} }
@ -687,8 +641,7 @@ static int32_t tsdbCloseCommitter(SCommitter2 *committer, int32_t eno) {
int32_t lino = 0; int32_t lino = 0;
if (eno == 0) { if (eno == 0) {
code = tsdbFSEditBegin(committer->tsdb->pFS, committer->fopArray, TSDB_FEDIT_COMMIT); TAOS_CHECK_GOTO(tsdbFSEditBegin(committer->tsdb->pFS, committer->fopArray, TSDB_FEDIT_COMMIT), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
// TODO // TODO
ASSERT(0); ASSERT(0);
@ -705,8 +658,8 @@ static int32_t tsdbCloseCommitter(SCommitter2 *committer, int32_t eno) {
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed at line %d since %s, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, lino, tsdbError("vgId:%d %s failed at %s:%d since %s, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, __FILE__,
tstrerror(code), committer->cid); lino, tstrerror(code), committer->cid);
} else { } else {
tsdbDebug("vgId:%d %s done, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, committer->cid); tsdbDebug("vgId:%d %s done, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, committer->cid);
} }
@ -740,22 +693,19 @@ int32_t tsdbCommitBegin(STsdb *tsdb, SCommitInfo *info) {
} else { } else {
SCommitter2 committer = {0}; SCommitter2 committer = {0};
code = tsdbOpenCommitter(tsdb, info, &committer); TAOS_CHECK_GOTO(tsdbOpenCommitter(tsdb, info, &committer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
for (int32_t i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) { for (int32_t i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) {
committer.ctx->info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i); committer.ctx->info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i);
code = tsdbCommitFileSet(&committer); TAOS_CHECK_GOTO(tsdbCommitFileSet(&committer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbCloseCommitter(&committer, code); TAOS_CHECK_GOTO(tsdbCloseCommitter(&committer, code), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} else { } else {
tsdbInfo("vgId:%d %s done, nRow:%" PRId64 " nDel:%" PRId64, TD_VID(tsdb->pVnode), __func__, nRow, nDel); tsdbInfo("vgId:%d %s done, nRow:%" PRId64 " nDel:%" PRId64, TD_VID(tsdb->pVnode), __func__, nRow, nDel);
} }
@ -792,7 +742,7 @@ int32_t tsdbCommitCommit(STsdb *tsdb) {
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code)); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} else { } else {
tsdbInfo("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); tsdbInfo("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__);
} }
@ -805,8 +755,7 @@ int32_t tsdbCommitAbort(STsdb *pTsdb) {
if (pTsdb->imem == NULL) goto _exit; if (pTsdb->imem == NULL) goto _exit;
code = tsdbFSEditAbort(pTsdb->pFS); TAOS_CHECK_GOTO(tsdbFSEditAbort(pTsdb->pFS), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
taosThreadMutexLock(&pTsdb->mutex); taosThreadMutexLock(&pTsdb->mutex);
for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) { for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) {

77
source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c
View File

@ -23,8 +23,7 @@ int32_t tsdbDataFileRAWReaderOpen(const char *fname, const SDataFileRAWReaderCon
reader[0] = taosMemoryCalloc(1, sizeof(SDataFileRAWReader)); reader[0] = taosMemoryCalloc(1, sizeof(SDataFileRAWReader));
if (reader[0] == NULL) { if (reader[0] == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
reader[0]->config[0] = config[0]; reader[0]->config[0] = config[0];
@ -32,25 +31,26 @@ int32_t tsdbDataFileRAWReaderOpen(const char *fname, const SDataFileRAWReaderCon
int32_t lcn = config->file.lcn; int32_t lcn = config->file.lcn;
if (fname) { if (fname) {
if (fname) { if (fname) {
code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd, lcn); TAOS_CHECK_GOTO(tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd, lcn), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} else { } else {
char fname1[TSDB_FILENAME_LEN]; char fname1[TSDB_FILENAME_LEN];
tsdbTFileName(config->tsdb, &config->file, fname1); tsdbTFileName(config->tsdb, &config->file, fname1);
code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd, lcn); TAOS_CHECK_GOTO(tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd, lcn), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
int32_t tsdbDataFileRAWReaderClose(SDataFileRAWReader **reader) { int32_t tsdbDataFileRAWReaderClose(SDataFileRAWReader **reader) {
if (reader[0] == NULL) return 0; if (reader[0] == NULL) {
return 0;
}
if (reader[0]->fd) { if (reader[0]->fd) {
tsdbCloseFile(&reader[0]->fd); tsdbCloseFile(&reader[0]->fd);
@ -75,12 +75,14 @@ int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlo
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength, 0, encryptAlgorithm, encryptKey); TAOS_CHECK_GOTO(
TSDB_CHECK_CODE(code, lino, _exit); tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength, 0, encryptAlgorithm, encryptKey),
&lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -91,18 +93,20 @@ int32_t tsdbDataFileRAWWriterOpen(const SDataFileRAWWriterConfig *config, SDataF
int32_t lino = 0; int32_t lino = 0;
SDataFileRAWWriter *writer = taosMemoryCalloc(1, sizeof(SDataFileRAWWriter)); SDataFileRAWWriter *writer = taosMemoryCalloc(1, sizeof(SDataFileRAWWriter));
if (!writer) return TSDB_CODE_OUT_OF_MEMORY; if (!writer) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
writer->config[0] = config[0]; writer->config[0] = config[0];
code = tsdbDataFileRAWWriterDoOpen(writer); TAOS_CHECK_GOTO(tsdbDataFileRAWWriterDoOpen(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
taosMemoryFree(writer); taosMemoryFree(writer);
writer = NULL; writer = NULL;
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
} }
ppWriter[0] = writer; ppWriter[0] = writer;
return code; return code;
@ -126,21 +130,20 @@ static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFil
.fid = writer->config->fid, .fid = writer->config->fid,
.nf = writer->file, .nf = writer->file,
}; };
code = TARRAY2_APPEND(opArr, op); TAOS_CHECK_GOTO(TARRAY2_APPEND(opArr, op), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
if (writer->fd) { if (writer->fd) {
code = tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey); TAOS_CHECK_GOTO(tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
tsdbCloseFile(&writer->fd); tsdbCloseFile(&writer->fd);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -157,12 +160,12 @@ static int32_t tsdbDataFileRAWWriterOpenDataFD(SDataFileRAWWriter *writer) {
} }
tsdbTFileName(writer->config->tsdb, &writer->file, fname); tsdbTFileName(writer->config->tsdb, &writer->file, fname);
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd, writer->file.lcn); TAOS_CHECK_GOTO(tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd, writer->file.lcn), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -174,30 +177,30 @@ int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer) {
writer->file = writer->config->file; writer->file = writer->config->file;
writer->ctx->offset = 0; writer->ctx->offset = 0;
code = tsdbDataFileRAWWriterOpenDataFD(writer); TAOS_CHECK_GOTO(tsdbDataFileRAWWriterOpenDataFD(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
writer->ctx->opened = true; writer->ctx->opened = true;
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr) { int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr) {
if (writer[0] == NULL) return 0; if (writer[0] == NULL) {
return 0;
}
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
if (writer[0]->ctx->opened) { if (writer[0]->ctx->opened) {
if (abort) { if (abort) {
code = tsdbDataFileRAWWriterCloseAbort(writer[0]); TAOS_CHECK_GOTO(tsdbDataFileRAWWriterCloseAbort(writer[0]), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
code = tsdbDataFileRAWWriterCloseCommit(writer[0], opArr); TAOS_CHECK_GOTO(tsdbDataFileRAWWriterCloseCommit(writer[0], opArr), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
tsdbDataFileRAWWriterDoClose(writer[0]); tsdbDataFileRAWWriterDoClose(writer[0]);
} }
@ -206,7 +209,8 @@ int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFil
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer[0]->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer[0]->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -216,15 +220,16 @@ int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDat
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
code = tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data, pDataBlock->dataLength, TAOS_CHECK_GOTO(tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data,
encryptAlgorithm, encryptKey); pDataBlock->dataLength, encryptAlgorithm, encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
writer->ctx->offset += pDataBlock->dataLength; writer->ctx->offset += pDataBlock->dataLength;
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }

655
source/dnode/vnode/src/tsdb/tsdbDataFileRW.c
View File

File diff suppressed because it is too large Load Diff

8
source/dnode/vnode/src/tsdb/tsdbDataFileRW.h
View File

@ -101,8 +101,8 @@ int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, uint32_t cmpr
int32_t encryptAlgorithm, char *encryptKey); int32_t encryptAlgorithm, char *encryptKey);
int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize, int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t encryptAlgorithm, char *encryptKey); int32_t encryptAlgorithm, char *encryptKey);
int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer, int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer, int32_t encryptAlgorithm,
int32_t encryptAlgorithm, char* encryptKey); char *encryptKey);
// tomb // tomb
int32_t tsdbDataFileWriteTombRecord(SDataFileWriter *writer, const STombRecord *record); int32_t tsdbDataFileWriteTombRecord(SDataFileWriter *writer, const STombRecord *record);
@ -111,8 +111,8 @@ int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAl
int32_t encryptAlgorithm, char *encryptKey); int32_t encryptAlgorithm, char *encryptKey);
int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize, int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t encryptAlgorithm, char *encryptKey); int32_t encryptAlgorithm, char *encryptKey);
int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize, int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm,
int32_t encryptAlgorithm, char* encryptKey); char *encryptKey);
// utils // utils
int32_t tsdbWriterUpdVerRange(SVersionRange *range, int64_t minVer, int64_t maxVer); int32_t tsdbWriterUpdVerRange(SVersionRange *range, int64_t minVer, int64_t maxVer);

76
source/dnode/vnode/src/tsdb/tsdbFile.c
View File

@ -150,74 +150,6 @@ void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, ch
bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2) { return pDelFile1->commitID == pDelFile2->commitID; } bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2) { return pDelFile1->commitID == pDelFile2->commitID; }
int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype) {
int32_t code = 0;
int64_t size = 0;
int64_t n;
TdFilePtr pFD;
char fname[TSDB_FILENAME_LEN] = {0};
char hdr[TSDB_FHDR_SIZE] = {0};
// truncate
switch (ftype) {
case TSDB_DATA_FILE:
size = pSet->pDataF->size;
tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname);
tPutDataFile(hdr, pSet->pDataF);
break;
case TSDB_SMA_FILE:
size = pSet->pSmaF->size;
tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
tPutSmaFile(hdr, pSet->pSmaF);
break;
default:
goto _err; // make the coverity scan happy
}
taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE);
// open
pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_WRITE_THROUGH);
if (pFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
// ftruncate
if (taosFtruncateFile(pFD, tsdbLogicToFileSize(size, pTsdb->pVnode->config.tsdbPageSize)) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
// update header
n = taosLSeekFile(pFD, 0, SEEK_SET);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
n = taosWriteFile(pFD, hdr, TSDB_FHDR_SIZE);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
// sync
if (taosFsyncFile(pFD) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
// close
taosCloseFile(&pFD);
return code;
_err:
tsdbError("vgId:%d, tsdb rollback file failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code;
}
int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) { int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) {
int32_t n = 0; int32_t n = 0;
@ -249,7 +181,7 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) {
// head // head
pSet->pHeadF = (SHeadFile *)taosMemoryCalloc(1, sizeof(SHeadFile)); pSet->pHeadF = (SHeadFile *)taosMemoryCalloc(1, sizeof(SHeadFile));
if (pSet->pHeadF == NULL) { if (pSet->pHeadF == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
pSet->pHeadF->nRef = 1; pSet->pHeadF->nRef = 1;
n += tGetHeadFile(p + n, pSet->pHeadF); n += tGetHeadFile(p + n, pSet->pHeadF);
@ -257,7 +189,7 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) {
// data // data
pSet->pDataF = (SDataFile *)taosMemoryCalloc(1, sizeof(SDataFile)); pSet->pDataF = (SDataFile *)taosMemoryCalloc(1, sizeof(SDataFile));
if (pSet->pDataF == NULL) { if (pSet->pDataF == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
pSet->pDataF->nRef = 1; pSet->pDataF->nRef = 1;
n += tGetDataFile(p + n, pSet->pDataF); n += tGetDataFile(p + n, pSet->pDataF);
@ -265,7 +197,7 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) {
// sma // sma
pSet->pSmaF = (SSmaFile *)taosMemoryCalloc(1, sizeof(SSmaFile)); pSet->pSmaF = (SSmaFile *)taosMemoryCalloc(1, sizeof(SSmaFile));
if (pSet->pSmaF == NULL) { if (pSet->pSmaF == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
pSet->pSmaF->nRef = 1; pSet->pSmaF->nRef = 1;
n += tGetSmaFile(p + n, pSet->pSmaF); n += tGetSmaFile(p + n, pSet->pSmaF);
@ -275,7 +207,7 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) {
for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
pSet->aSttF[iStt] = (SSttFile *)taosMemoryCalloc(1, sizeof(SSttFile)); pSet->aSttF[iStt] = (SSttFile *)taosMemoryCalloc(1, sizeof(SSttFile));
if (pSet->aSttF[iStt] == NULL) { if (pSet->aSttF[iStt] == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
pSet->aSttF[iStt]->nRef = 1; pSet->aSttF[iStt]->nRef = 1;
n += tGetSttFile(p + n, pSet->aSttF[iStt]); n += tGetSttFile(p + n, pSet->aSttF[iStt]);

20
source/dnode/vnode/src/tsdb/tsdbFile2.c
View File

@ -165,8 +165,7 @@ static int32_t data_to_json(const STFile *file, cJSON *json) { return tfile_to_j
static int32_t sma_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); } static int32_t sma_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); }
static int32_t tomb_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); } static int32_t tomb_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); }
static int32_t stt_to_json(const STFile *file, cJSON *json) { static int32_t stt_to_json(const STFile *file, cJSON *json) {
int32_t code = tfile_to_json(file, json); TAOS_CHECK_RETURN(tfile_to_json(file, json));
if (code) return code;
/* lvl */ /* lvl */
if (cJSON_AddNumberToObject(json, "level", file->stt->level) == NULL) { if (cJSON_AddNumberToObject(json, "level", file->stt->level) == NULL) {
@ -181,8 +180,7 @@ static int32_t data_from_json(const cJSON *json, STFile *file) { return tfile_fr
static int32_t sma_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); } static int32_t sma_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); }
static int32_t tomb_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); } static int32_t tomb_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); }
static int32_t stt_from_json(const cJSON *json, STFile *file) { static int32_t stt_from_json(const cJSON *json, STFile *file) {
int32_t code = tfile_from_json(json, file); TAOS_CHECK_RETURN(tfile_from_json(json, file));
if (code) return code;
const cJSON *item; const cJSON *item;
@ -202,7 +200,9 @@ int32_t tsdbTFileToJson(const STFile *file, cJSON *json) {
return g_tfile_info[file->type].to_json(file, json); return g_tfile_info[file->type].to_json(file, json);
} else { } else {
cJSON *item = cJSON_AddObjectToObject(json, g_tfile_info[file->type].suffix); cJSON *item = cJSON_AddObjectToObject(json, g_tfile_info[file->type].suffix);
if (item == NULL) return TSDB_CODE_OUT_OF_MEMORY; if (item == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
return g_tfile_info[file->type].to_json(file, item); return g_tfile_info[file->type].to_json(file, item);
} }
} }
@ -211,13 +211,11 @@ int32_t tsdbJsonToTFile(const cJSON *json, tsdb_ftype_t ftype, STFile *f) {
f[0] = (STFile){.type = ftype}; f[0] = (STFile){.type = ftype};
if (ftype == TSDB_FTYPE_STT) { if (ftype == TSDB_FTYPE_STT) {
int32_t code = g_tfile_info[ftype].from_json(json, f); TAOS_CHECK_RETURN(g_tfile_info[ftype].from_json(json, f));
if (code) return code;
} else { } else {
const cJSON *item = cJSON_GetObjectItem(json, g_tfile_info[ftype].suffix); const cJSON *item = cJSON_GetObjectItem(json, g_tfile_info[ftype].suffix);
if (cJSON_IsObject(item)) { if (cJSON_IsObject(item)) {
int32_t code = g_tfile_info[ftype].from_json(item, f); TAOS_CHECK_RETURN(g_tfile_info[ftype].from_json(item, f));
if (code) return code;
} else { } else {
return TSDB_CODE_NOT_FOUND; return TSDB_CODE_NOT_FOUND;
} }
@ -228,7 +226,9 @@ int32_t tsdbJsonToTFile(const cJSON *json, tsdb_ftype_t ftype, STFile *f) {
int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) { int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) {
fobj[0] = taosMemoryMalloc(sizeof(*fobj[0])); fobj[0] = taosMemoryMalloc(sizeof(*fobj[0]));
if (!fobj[0]) return TSDB_CODE_OUT_OF_MEMORY; if (!fobj[0]) {
return TSDB_CODE_OUT_OF_MEMORY;
}
taosThreadMutexInit(&fobj[0]->mutex, NULL); taosThreadMutexInit(&fobj[0]->mutex, NULL);
fobj[0]->f[0] = f[0]; fobj[0]->f[0] = f[0];

171
source/dnode/vnode/src/tsdb/tsdbMerge.c
View File

@ -66,7 +66,6 @@ static int32_t tsdbMergerOpen(SMerger *merger) {
} }
static int32_t tsdbMergerClose(SMerger *merger) { static int32_t tsdbMergerClose(SMerger *merger) {
int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
SVnode *pVnode = merger->tsdb->pVnode; SVnode *pVnode = merger->tsdb->pVnode;
@ -82,12 +81,7 @@ static int32_t tsdbMergerClose(SMerger *merger) {
TARRAY2_DESTROY(merger->dataIterArr, NULL); TARRAY2_DESTROY(merger->dataIterArr, NULL);
TARRAY2_DESTROY(merger->sttReaderArr, NULL); TARRAY2_DESTROY(merger->sttReaderArr, NULL);
TARRAY2_DESTROY(merger->fopArr, NULL); TARRAY2_DESTROY(merger->fopArr, NULL);
return 0;
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pVnode), lino, code);
}
return code;
} }
static int32_t tsdbMergeFileSetEndCloseReader(SMerger *merger) { static int32_t tsdbMergeFileSetEndCloseReader(SMerger *merger) {
@ -126,8 +120,7 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) {
.fid = merger->ctx->fset->fid, .fid = merger->ctx->fset->fid,
.of = fobj->f[0], .of = fobj->f[0],
}; };
code = TARRAY2_APPEND(merger->fopArr, op); TAOS_CHECK_GOTO(TARRAY2_APPEND(merger->fopArr, op), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
SSttFileReader *reader; SSttFileReader *reader;
SSttFileReaderConfig config = { SSttFileReaderConfig config = {
@ -136,11 +129,9 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) {
.file[0] = fobj->f[0], .file[0] = fobj->f[0],
}; };
code = tsdbSttFileReaderOpen(fobj->fname, &config, &reader); TAOS_CHECK_GOTO(tsdbSttFileReaderOpen(fobj->fname, &config, &reader), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = TARRAY2_APPEND(merger->sttReaderArr, reader); TAOS_CHECK_GOTO(TARRAY2_APPEND(merger->sttReaderArr, reader), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
} else { } else {
@ -212,8 +203,7 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) {
.fid = merger->ctx->fset->fid, .fid = merger->ctx->fset->fid,
.of = fobj->f[0], .of = fobj->f[0],
}; };
code = TARRAY2_APPEND(merger->fopArr, op); TAOS_CHECK_GOTO(TARRAY2_APPEND(merger->fopArr, op), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
SSttFileReader *reader; SSttFileReader *reader;
SSttFileReaderConfig config = { SSttFileReaderConfig config = {
@ -222,8 +212,7 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) {
.file[0] = fobj->f[0], .file[0] = fobj->f[0],
}; };
code = tsdbSttFileReaderOpen(fobj->fname, &config, &reader); TAOS_CHECK_GOTO(tsdbSttFileReaderOpen(fobj->fname, &config, &reader), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
if ((code = TARRAY2_APPEND(merger->sttReaderArr, reader))) { if ((code = TARRAY2_APPEND(merger->sttReaderArr, reader))) {
tsdbSttFileReaderClose(&reader); tsdbSttFileReaderClose(&reader);
@ -239,8 +228,9 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) {
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(merger->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
tsdbMergeFileSetEndCloseReader(merger); tsdbMergeFileSetEndCloseReader(merger);
TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code);
} }
return code; return code;
} }
@ -259,32 +249,25 @@ static int32_t tsdbMergeFileSetBeginOpenIter(SMerger *merger) {
config.type = TSDB_ITER_TYPE_STT; config.type = TSDB_ITER_TYPE_STT;
config.sttReader = sttReader; config.sttReader = sttReader;
code = tsdbIterOpen(&config, &iter); TAOS_CHECK_GOTO(tsdbIterOpen(&config, &iter), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(TARRAY2_APPEND(merger->dataIterArr, iter), &lino, _exit);
code = TARRAY2_APPEND(merger->dataIterArr, iter);
TSDB_CHECK_CODE(code, lino, _exit);
// tomb iter // tomb iter
config.type = TSDB_ITER_TYPE_STT_TOMB; config.type = TSDB_ITER_TYPE_STT_TOMB;
config.sttReader = sttReader; config.sttReader = sttReader;
code = tsdbIterOpen(&config, &iter); TAOS_CHECK_GOTO(tsdbIterOpen(&config, &iter), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = TARRAY2_APPEND(merger->tombIterArr, iter); TAOS_CHECK_GOTO(TARRAY2_APPEND(merger->tombIterArr, iter), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbIterMergerOpen(merger->dataIterArr, &merger->dataIterMerger, false); TAOS_CHECK_GOTO(tsdbIterMergerOpen(merger->dataIterArr, &merger->dataIterMerger, false), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbIterMergerOpen(merger->tombIterArr, &merger->tombIterMerger, true); TAOS_CHECK_GOTO(tsdbIterMergerOpen(merger->tombIterArr, &merger->tombIterMerger, true), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(vid, lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", vid, __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -296,10 +279,9 @@ static int32_t tsdbMergeFileSetBeginOpenWriter(SMerger *merger) {
SDiskID did; SDiskID did;
int32_t level = tsdbFidLevel(merger->ctx->fset->fid, &merger->tsdb->keepCfg, merger->ctx->now); int32_t level = tsdbFidLevel(merger->ctx->fset->fid, &merger->tsdb->keepCfg, merger->ctx->now);
if (tfsAllocDisk(merger->tsdb->pVnode->pTfs, level, &did) < 0) {
code = TSDB_CODE_FS_NO_VALID_DISK; TAOS_CHECK_GOTO(tfsAllocDisk(merger->tsdb->pVnode->pTfs, level, &did), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
}
tfsMkdirRecurAt(merger->tsdb->pVnode->pTfs, merger->tsdb->path, did); tfsMkdirRecurAt(merger->tsdb->pVnode->pTfs, merger->tsdb->path, did);
SFSetWriterConfig config = { SFSetWriterConfig config = {
.tsdb = merger->tsdb, .tsdb = merger->tsdb,
@ -328,12 +310,11 @@ static int32_t tsdbMergeFileSetBeginOpenWriter(SMerger *merger) {
} }
} }
code = tsdbFSetWriterOpen(&config, &merger->writer); TAOS_CHECK_GOTO(tsdbFSetWriterOpen(&config, &merger->writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(vid, lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", vid, __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -353,20 +334,18 @@ static int32_t tsdbMergeFileSetBegin(SMerger *merger) {
merger->ctx->tbid->uid = 0; merger->ctx->tbid->uid = 0;
// open reader // open reader
code = tsdbMergeFileSetBeginOpenReader(merger); TAOS_CHECK_GOTO(tsdbMergeFileSetBeginOpenReader(merger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// open iterator // open iterator
code = tsdbMergeFileSetBeginOpenIter(merger); TAOS_CHECK_GOTO(tsdbMergeFileSetBeginOpenIter(merger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// open writer // open writer
code = tsdbMergeFileSetBeginOpenWriter(merger); TAOS_CHECK_GOTO(tsdbMergeFileSetBeginOpenWriter(merger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(merger->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -387,18 +366,14 @@ static int32_t tsdbMergeFileSetEnd(SMerger *merger) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
code = tsdbMergeFileSetEndCloseWriter(merger); TAOS_CHECK_GOTO(tsdbMergeFileSetEndCloseWriter(merger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbMergeFileSetEndCloseIter(merger); TAOS_CHECK_GOTO(tsdbMergeFileSetEndCloseIter(merger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbMergeFileSetEndCloseReader(merger); TAOS_CHECK_GOTO(tsdbMergeFileSetEndCloseReader(merger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// edit file system // edit file system
code = tsdbFSEditBegin(merger->tsdb->pFS, merger->fopArr, TSDB_FEDIT_MERGE); TAOS_CHECK_GOTO(tsdbFSEditBegin(merger->tsdb->pFS, merger->fopArr, TSDB_FEDIT_MERGE), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
taosThreadMutexLock(&merger->tsdb->mutex); taosThreadMutexLock(&merger->tsdb->mutex);
code = tsdbFSEditCommit(merger->tsdb->pFS); code = tsdbFSEditCommit(merger->tsdb->pFS);
@ -410,6 +385,8 @@ static int32_t tsdbMergeFileSetEnd(SMerger *merger) {
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(merger->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code); TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code);
} }
return code; return code;
@ -420,8 +397,7 @@ static int32_t tsdbMergeFileSet(SMerger *merger, STFileSet *fset) {
int32_t lino = 0; int32_t lino = 0;
merger->ctx->fset = fset; merger->ctx->fset = fset;
code = tsdbMergeFileSetBegin(merger); TAOS_CHECK_GOTO(tsdbMergeFileSetBegin(merger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// data // data
SMetaInfo info; SMetaInfo info;
@ -434,17 +410,14 @@ static int32_t tsdbMergeFileSet(SMerger *merger, STFileSet *fset) {
merger->ctx->tbid->suid = row->suid; merger->ctx->tbid->suid = row->suid;
if (metaGetInfo(merger->tsdb->pVnode->pMeta, row->uid, &info, NULL) != 0) { if (metaGetInfo(merger->tsdb->pVnode->pMeta, row->uid, &info, NULL) != 0) {
code = tsdbIterMergerSkipTableData(merger->dataIterMerger, merger->ctx->tbid); TAOS_CHECK_GOTO(tsdbIterMergerSkipTableData(merger->dataIterMerger, merger->ctx->tbid), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
continue; continue;
} }
} }
code = tsdbFSetWriteRow(merger->writer, row); TAOS_CHECK_GOTO(tsdbFSetWriteRow(merger->writer, row), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbIterMergerNext(merger->dataIterMerger); TAOS_CHECK_GOTO(tsdbIterMergerNext(merger->dataIterMerger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
// tomb // tomb
@ -456,24 +429,21 @@ static int32_t tsdbMergeFileSet(SMerger *merger, STFileSet *fset) {
merger->ctx->tbid->suid = record->suid; merger->ctx->tbid->suid = record->suid;
if (metaGetInfo(merger->tsdb->pVnode->pMeta, record->uid, &info, NULL) != 0) { if (metaGetInfo(merger->tsdb->pVnode->pMeta, record->uid, &info, NULL) != 0) {
code = tsdbIterMergerSkipTableData(merger->tombIterMerger, merger->ctx->tbid); TAOS_CHECK_GOTO(tsdbIterMergerSkipTableData(merger->tombIterMerger, merger->ctx->tbid), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
continue; continue;
} }
} }
code = tsdbFSetWriteTombRecord(merger->writer, record); TAOS_CHECK_GOTO(tsdbFSetWriteTombRecord(merger->writer, record), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbIterMergerNext(merger->tombIterMerger); TAOS_CHECK_GOTO(tsdbIterMergerNext(merger->tombIterMerger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbMergeFileSetEnd(merger); TAOS_CHECK_GOTO(tsdbMergeFileSetEnd(merger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(merger->tsdb->pVnode), __func__, lino, tstrerror(code)); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(merger->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} else { } else {
tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(merger->tsdb->pVnode), __func__, fset->fid); tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(merger->tsdb->pVnode), __func__, fset->fid);
} }
@ -487,20 +457,18 @@ static int32_t tsdbDoMerge(SMerger *merger) {
if (TARRAY2_SIZE(merger->fset->lvlArr) == 0) return 0; if (TARRAY2_SIZE(merger->fset->lvlArr) == 0) return 0;
SSttLvl *lvl = TARRAY2_FIRST(merger->fset->lvlArr); SSttLvl *lvl = TARRAY2_FIRST(merger->fset->lvlArr);
if (lvl->level != 0 || TARRAY2_SIZE(lvl->fobjArr) < merger->sttTrigger) return 0; if (lvl->level != 0 || TARRAY2_SIZE(lvl->fobjArr) < merger->sttTrigger) {
return 0;
}
code = tsdbMergerOpen(merger); TAOS_CHECK_GOTO(tsdbMergerOpen(merger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tsdbMergeFileSet(merger, merger->fset), &lino, _exit);
TAOS_CHECK_GOTO(tsdbMergerClose(merger), &lino, _exit);
code = tsdbMergeFileSet(merger, merger->fset);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbMergerClose(merger);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(merger->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} else { } else {
tsdbDebug("vgId:%d %s done", TD_VID(merger->tsdb->pVnode), __func__); tsdbDebug("vgId:%d %s done", TD_VID(merger->tsdb->pVnode), __func__);
} }
@ -543,45 +511,12 @@ int32_t tsdbMerge(void *arg) {
if (merger->sttTrigger <= 1) return 0; if (merger->sttTrigger <= 1) return 0;
// copy snapshot // copy snapshot
code = tsdbMergeGetFSet(merger); TAOS_CHECK_GOTO(tsdbMergeGetFSet(merger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
if (merger->fset == NULL) return 0; if (merger->fset == NULL) {
/* return 0;
bool skipMerge = false;
{
extern int8_t tsS3Enabled;
extern int32_t tsS3UploadDelaySec;
long s3Size(const char *object_name);
int32_t nlevel = tfsGetLevel(merger->tsdb->pVnode->pTfs);
if (tsS3Enabled && nlevel > 1) {
STFileObj *fobj = merger->fset->farr[TSDB_FTYPE_DATA];
if (fobj && fobj->f->did.level == nlevel - 1) {
// if exists on s3 or local mtime < committer->ctx->now - tsS3UploadDelay
const char *object_name = taosDirEntryBaseName((char *)fobj->fname);
if (taosCheckExistFile(fobj->fname)) {
int32_t now = taosGetTimestampSec();
int32_t mtime = 0;
taosStatFile(fobj->fname, NULL, &mtime, NULL);
if (mtime < now - tsS3UploadDelaySec) {
skipMerge = true;
}
} else // if (s3Size(object_name) > 0)
{
skipMerge = true;
}
}
// new fset can be written with ts data
}
} }
if (skipMerge) {
code = 0;
goto _exit;
}
*/
// do merge // do merge
tsdbInfo("vgId:%d merge begin, fid:%d", TD_VID(tsdb->pVnode), merger->fid); tsdbInfo("vgId:%d merge begin, fid:%d", TD_VID(tsdb->pVnode), merger->fid);
code = tsdbDoMerge(merger); code = tsdbDoMerge(merger);
@ -590,7 +525,7 @@ int32_t tsdbMerge(void *arg) {
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
tsdbFatal("vgId:%d, failed to merge stt files since %s. code:%d", TD_VID(tsdb->pVnode), terrstr(), code); tsdbFatal("vgId:%d, failed to merge stt files since %s. code:%d", TD_VID(tsdb->pVnode), terrstr(), code);
taosMsleep(100); taosMsleep(100);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);

53
source/dnode/vnode/src/tsdb/tsdbOpen.c
View File

@ -38,17 +38,11 @@ int64_t tsdbGetEarliestTs(STsdb *pTsdb) {
return ts; return ts;
} }
/** int32_t tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKeepCfg, int8_t rollback, bool force) {
* @brief
*
* @param pVnode
* @param ppTsdb
* @param dir
* @return int 0: success, -1: failed
*/
int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKeepCfg, int8_t rollback, bool force) {
STsdb *pTsdb = NULL; STsdb *pTsdb = NULL;
int slen = 0; int slen = 0;
int32_t code;
int32_t lino;
*ppTsdb = NULL; *ppTsdb = NULL;
slen = strlen(pVnode->path) + strlen(dir) + 2; slen = strlen(pVnode->path) + strlen(dir) + 2;
@ -56,8 +50,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
// create handle // create handle
pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen); pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen);
if (pTsdb == NULL) { if (pTsdb == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_RETURN(TSDB_CODE_OUT_OF_MEMORY);
return -1;
} }
pTsdb->path = (char *)&pTsdb[1]; pTsdb->path = (char *)&pTsdb[1];
@ -79,42 +72,34 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
} }
// open tsdb // open tsdb
int32_t code = tsdbOpenFS(pTsdb, &pTsdb->pFS, rollback); TAOS_CHECK_GOTO(tsdbOpenFS(pTsdb, &pTsdb->pFS, rollback), &lino, _exit);
if (code < 0) {
terrno = code;
goto _err;
}
if (pTsdb->pFS->fsstate == TSDB_FS_STATE_INCOMPLETE && force == false) { if (pTsdb->pFS->fsstate == TSDB_FS_STATE_INCOMPLETE && force == false) {
terrno = TSDB_CODE_NEED_RETRY; TAOS_CHECK_GOTO(TSDB_CODE_NEED_RETRY, &lino, _exit);
goto _err;
} }
if (tsdbOpenCache(pTsdb) < 0) { TAOS_CHECK_GOTO(tsdbOpenCache(pTsdb), &lino, _exit);
goto _err;
}
#ifdef TD_ENTERPRISE #ifdef TD_ENTERPRISE
if (tsdbOpenCompMonitor(pTsdb) < 0) { TAOS_CHECK_GOTO(tsdbOpenCompMonitor(pTsdb), &lino, _exit);
goto _err;
}
#endif #endif
tsdbDebug("vgId:%d, tsdb is opened at %s, days:%d, keep:%d,%d,%d, keepTimeoffset:%d", TD_VID(pVnode), pTsdb->path, _exit:
pTsdb->keepCfg.days, pTsdb->keepCfg.keep0, pTsdb->keepCfg.keep1, pTsdb->keepCfg.keep2, if (code) {
pTsdb->keepCfg.keepTimeOffset); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, lino, tstrerror(code));
*ppTsdb = pTsdb;
return 0;
_err:
tsdbCloseFS(&pTsdb->pFS); tsdbCloseFS(&pTsdb->pFS);
taosThreadMutexDestroy(&pTsdb->mutex); taosThreadMutexDestroy(&pTsdb->mutex);
taosMemoryFree(pTsdb); taosMemoryFree(pTsdb);
return -1; } else {
tsdbDebug("vgId:%d, tsdb is opened at %s, days:%d, keep:%d,%d,%d, keepTimeoffset:%d", TD_VID(pVnode), pTsdb->path,
pTsdb->keepCfg.days, pTsdb->keepCfg.keep0, pTsdb->keepCfg.keep1, pTsdb->keepCfg.keep2,
pTsdb->keepCfg.keepTimeOffset);
*ppTsdb = pTsdb;
}
return code;
} }
int tsdbClose(STsdb **pTsdb) { int32_t tsdbClose(STsdb **pTsdb) {
if (*pTsdb) { if (*pTsdb) {
STsdb *pdb = *pTsdb; STsdb *pdb = *pTsdb;
tsdbDebug("vgId:%d, tsdb is close at %s, days:%d, keep:%d,%d,%d, keepTimeOffset:%d", TD_VID(pdb->pVnode), pdb->path, tsdbDebug("vgId:%d, tsdb is close at %s, days:%d, keep:%d,%d,%d, keepTimeOffset:%d", TD_VID(pdb->pVnode), pdb->path,

381
source/dnode/vnode/src/tsdb/tsdbSttFileRW.c
View File

@ -40,7 +40,9 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con
int32_t lino = 0; int32_t lino = 0;
reader[0] = taosMemoryCalloc(1, sizeof(*reader[0])); reader[0] = taosMemoryCalloc(1, sizeof(*reader[0]));
if (reader[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; if (reader[0] == NULL) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
reader[0]->config[0] = config[0]; reader[0]->config[0] = config[0];
reader[0]->buffers = config->buffers; reader[0]->buffers = config->buffers;
@ -50,13 +52,11 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con
// open file // open file
if (fname) { if (fname) {
code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0); TAOS_CHECK_GOTO(tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
char fname1[TSDB_FILENAME_LEN]; char fname1[TSDB_FILENAME_LEN];
tsdbTFileName(config->tsdb, config->file, fname1); tsdbTFileName(config->tsdb, config->file, fname1);
code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0); TAOS_CHECK_GOTO(tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
// // open each segment reader // // open each segment reader
@ -66,15 +66,15 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con
int32_t encryptAlgoirthm = config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgoirthm = config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = config->tsdb->pVnode->config.tsdbCfg.encryptKey;
#if 1 #if 1
code = tsdbReadFile(reader[0]->fd, offset, (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0, encryptAlgoirthm, TAOS_CHECK_GOTO(tsdbReadFile(reader[0]->fd, offset, (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0,
encryptKey); encryptAlgoirthm, encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
#else #else
int64_t size = config->file->size; int64_t size = config->file->size;
for (; size > TSDB_FHDR_SIZE; size--) { for (; size > TSDB_FHDR_SIZE; size--) {
code = tsdbReadFile(reader[0]->fd, size - sizeof(SSttFooter), (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0, encryptAlgoirthm, code = tsdbReadFile(reader[0]->fd, size - sizeof(SSttFooter), (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0,
encryptKey); encryptAlgoirthm, encryptKey);
if (code) continue; if (code) continue;
if ((*reader)->footer->sttBlkPtr->offset + (*reader)->footer->sttBlkPtr->size + sizeof(SSttFooter) == size || if ((*reader)->footer->sttBlkPtr->offset + (*reader)->footer->sttBlkPtr->size + sizeof(SSttFooter) == size ||
(*reader)->footer->statisBlkPtr->offset + (*reader)->footer->statisBlkPtr->size + sizeof(SSttFooter) == size || (*reader)->footer->statisBlkPtr->offset + (*reader)->footer->statisBlkPtr->size + sizeof(SSttFooter) == size ||
@ -89,7 +89,8 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
tsdbSttFileReaderClose(reader); tsdbSttFileReaderClose(reader);
} }
return code; return code;
@ -118,13 +119,14 @@ int32_t tsdbSttFileReadStatisBlk(SSttFileReader *reader, const TStatisBlkArray *
int32_t size = reader->footer->statisBlkPtr->size / sizeof(SStatisBlk); int32_t size = reader->footer->statisBlkPtr->size / sizeof(SStatisBlk);
void *data = taosMemoryMalloc(reader->footer->statisBlkPtr->size); void *data = taosMemoryMalloc(reader->footer->statisBlkPtr->size);
if (!data) return TSDB_CODE_OUT_OF_MEMORY; if (!data) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
int32_t code = int32_t code = tsdbReadFile(reader->fd, reader->footer->statisBlkPtr->offset, data,
tsdbReadFile(reader->fd, reader->footer->statisBlkPtr->offset, data, reader->footer->statisBlkPtr->size, 0, reader->footer->statisBlkPtr->size, 0, encryptAlgorithm, encryptKey);
encryptAlgorithm, encryptKey);
if (code) { if (code) {
taosMemoryFree(data); taosMemoryFree(data);
return code; return code;
@ -149,13 +151,14 @@ int32_t tsdbSttFileReadTombBlk(SSttFileReader *reader, const TTombBlkArray **tom
int32_t size = reader->footer->tombBlkPtr->size / sizeof(STombBlk); int32_t size = reader->footer->tombBlkPtr->size / sizeof(STombBlk);
void *data = taosMemoryMalloc(reader->footer->tombBlkPtr->size); void *data = taosMemoryMalloc(reader->footer->tombBlkPtr->size);
if (!data) return TSDB_CODE_OUT_OF_MEMORY; if (!data) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
int32_t code = int32_t code = tsdbReadFile(reader->fd, reader->footer->tombBlkPtr->offset, data,
tsdbReadFile(reader->fd, reader->footer->tombBlkPtr->offset, data, reader->footer->tombBlkPtr->size, 0, reader->footer->tombBlkPtr->size, 0, encryptAlgorithm, encryptKey);
encryptAlgorithm, encryptKey);
if (code) { if (code) {
taosMemoryFree(data); taosMemoryFree(data);
return code; return code;
@ -180,13 +183,14 @@ int32_t tsdbSttFileReadSttBlk(SSttFileReader *reader, const TSttBlkArray **sttBl
int32_t size = reader->footer->sttBlkPtr->size / sizeof(SSttBlk); int32_t size = reader->footer->sttBlkPtr->size / sizeof(SSttBlk);
void *data = taosMemoryMalloc(reader->footer->sttBlkPtr->size); void *data = taosMemoryMalloc(reader->footer->sttBlkPtr->size);
if (!data) return TSDB_CODE_OUT_OF_MEMORY; if (!data) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
int32_t code = int32_t code = tsdbReadFile(reader->fd, reader->footer->sttBlkPtr->offset, data, reader->footer->sttBlkPtr->size,
tsdbReadFile(reader->fd, reader->footer->sttBlkPtr->offset, data, reader->footer->sttBlkPtr->size, 0, 0, encryptAlgorithm, encryptKey);
encryptAlgorithm, encryptKey);
if (code) { if (code) {
taosMemoryFree(data); taosMemoryFree(data);
return code; return code;
@ -215,17 +219,17 @@ int32_t tsdbSttFileReadBlockData(SSttFileReader *reader, const SSttBlk *sttBlk,
char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load data // load data
tBufferClear(buffer0); tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szBlock, buffer0, 0, TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szBlock, buffer0, 0,
encryptAlgorithm, encryptKey); encryptAlgorithm, encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0); SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0);
code = tBlockDataDecompress(&br, bData, assist); TAOS_CHECK_GOTO(tBlockDataDecompress(&br, bData, assist), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -244,14 +248,13 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load key part // load key part
tBufferClear(buffer0); tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szKey, buffer0, 0, TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szKey, buffer0, 0,
encryptAlgorithm, encryptKey); encryptAlgorithm, encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
// decode header // decode header
SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0); SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0);
code = tGetDiskDataHdr(&br, &hdr); TAOS_CHECK_GOTO(tGetDiskDataHdr(&br, &hdr), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
ASSERT(hdr.delimiter == TSDB_FILE_DLMT); ASSERT(hdr.delimiter == TSDB_FILE_DLMT);
@ -262,8 +265,7 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
bData->nRow = hdr.nRow; bData->nRow = hdr.nRow;
// key part // key part
code = tBlockDataDecompressKeyPart(&hdr, &br, bData, assist); TAOS_CHECK_GOTO(tBlockDataDecompressKeyPart(&hdr, &br, bData, assist), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
ASSERT(br.offset == buffer0->size); ASSERT(br.offset == buffer0->size);
bool loadExtra = false; bool loadExtra = false;
@ -280,9 +282,9 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
// load SBlockCol part // load SBlockCol part
tBufferClear(buffer0); tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey, hdr.szBlkCol, buffer0, 0, TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey, hdr.szBlkCol, buffer0, 0,
encryptAlgorithm, encryptKey); encryptAlgorithm, encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
// load each column // load each column
SBlockCol blockCol = { SBlockCol blockCol = {
@ -302,8 +304,7 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
break; break;
} }
code = tGetBlockCol(&br, &blockCol, hdr.fmtVer, hdr.cmprAlg); TAOS_CHECK_GOTO(tGetBlockCol(&br, &blockCol, hdr.fmtVer, hdr.cmprAlg), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (cid < blockCol.cid) { if (cid < blockCol.cid) {
@ -320,27 +321,26 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
.szValue = 0, .szValue = 0,
.offset = 0, .offset = 0,
}; };
code = tBlockDataDecompressColData(&hdr, &none, &br, bData, assist); TAOS_CHECK_GOTO(tBlockDataDecompressColData(&hdr, &none, &br, bData, assist), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} else if (cid == blockCol.cid) { } else if (cid == blockCol.cid) {
// load from file // load from file
tBufferClear(buffer1); tBufferClear(buffer1);
code = TAOS_CHECK_GOTO(
tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey + hdr.szBlkCol + blockCol.offset, tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey + hdr.szBlkCol + blockCol.offset,
blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, 0, blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, 0, encryptAlgorithm,
encryptAlgorithm, encryptKey); encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
// decode the buffer // decode the buffer
SBufferReader br1 = BUFFER_READER_INITIALIZER(0, buffer1); SBufferReader br1 = BUFFER_READER_INITIALIZER(0, buffer1);
code = tBlockDataDecompressColData(&hdr, &blockCol, &br1, bData, assist); TAOS_CHECK_GOTO(tBlockDataDecompressColData(&hdr, &blockCol, &br1, bData, assist), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -356,9 +356,9 @@ int32_t tsdbSttFileReadTombBlock(SSttFileReader *reader, const STombBlk *tombBlk
char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load // load
tBufferClear(buffer0); tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd, tombBlk->dp->offset, tombBlk->dp->size, buffer0, 0, TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd, tombBlk->dp->offset, tombBlk->dp->size, buffer0, 0, encryptAlgorithm,
encryptAlgorithm, encryptKey); encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
// decode // decode
int32_t size = 0; int32_t size = 0;
@ -372,15 +372,15 @@ int32_t tsdbSttFileReadTombBlock(SSttFileReader *reader, const STombBlk *tombBlk
.originalSize = tombBlk->numRec * sizeof(int64_t), .originalSize = tombBlk->numRec * sizeof(int64_t),
.compressedSize = tombBlk->size[i], .compressedSize = tombBlk->size[i],
}; };
code = tDecompressDataToBuffer(BR_PTR(&br), &cinfo, tombBlock->buffers + i, assist); TAOS_CHECK_GOTO(tDecompressDataToBuffer(BR_PTR(&br), &cinfo, tombBlock->buffers + i, assist), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
br.offset += tombBlk->size[i]; br.offset += tombBlk->size[i];
} }
ASSERT(br.offset == tombBlk->dp->size); ASSERT(br.offset == tombBlk->dp->size);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -396,9 +396,9 @@ int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *sta
char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load data // load data
tBufferClear(buffer0); tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd, statisBlk->dp->offset, statisBlk->dp->size, buffer0, 0, TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd, statisBlk->dp->offset, statisBlk->dp->size, buffer0, 0,
encryptAlgorithm, encryptKey); encryptAlgorithm, encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
// decode data // decode data
tStatisBlockClear(statisBlock); tStatisBlockClear(statisBlock);
@ -413,8 +413,7 @@ int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *sta
.originalSize = statisBlk->numRec * sizeof(int64_t), .originalSize = statisBlk->numRec * sizeof(int64_t),
}; };
code = tDecompressDataToBuffer(BR_PTR(&br), &info, &statisBlock->buffers[i], assist); TAOS_CHECK_GOTO(tDecompressDataToBuffer(BR_PTR(&br), &info, &statisBlock->buffers[i], assist), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
br.offset += statisBlk->size[i]; br.offset += statisBlk->size[i];
} }
@ -424,25 +423,23 @@ int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *sta
// decode compress info // decode compress info
for (int32_t i = 0; i < statisBlk->numOfPKs; i++) { for (int32_t i = 0; i < statisBlk->numOfPKs; i++) {
code = tValueColumnCompressInfoDecode(&br, &firstKeyInfos[i]); TAOS_CHECK_GOTO(tValueColumnCompressInfoDecode(&br, &firstKeyInfos[i]), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
for (int32_t i = 0; i < statisBlk->numOfPKs; i++) { for (int32_t i = 0; i < statisBlk->numOfPKs; i++) {
code = tValueColumnCompressInfoDecode(&br, &lastKeyInfos[i]); TAOS_CHECK_GOTO(tValueColumnCompressInfoDecode(&br, &lastKeyInfos[i]), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
// decode value columns // decode value columns
for (int32_t i = 0; i < statisBlk->numOfPKs; i++) { for (int32_t i = 0; i < statisBlk->numOfPKs; i++) {
code = tValueColumnDecompress(BR_PTR(&br), firstKeyInfos + i, &statisBlock->firstKeyPKs[i], assist); TAOS_CHECK_GOTO(tValueColumnDecompress(BR_PTR(&br), firstKeyInfos + i, &statisBlock->firstKeyPKs[i], assist),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
br.offset += (firstKeyInfos[i].dataCompressedSize + firstKeyInfos[i].offsetCompressedSize); br.offset += (firstKeyInfos[i].dataCompressedSize + firstKeyInfos[i].offsetCompressedSize);
} }
for (int32_t i = 0; i < statisBlk->numOfPKs; i++) { for (int32_t i = 0; i < statisBlk->numOfPKs; i++) {
code = tValueColumnDecompress(BR_PTR(&br), &lastKeyInfos[i], &statisBlock->lastKeyPKs[i], assist); TAOS_CHECK_GOTO(tValueColumnDecompress(BR_PTR(&br), &lastKeyInfos[i], &statisBlock->lastKeyPKs[i], assist), &lino,
TSDB_CHECK_CODE(code, lino, _exit); _exit);
br.offset += (lastKeyInfos[i].dataCompressedSize + lastKeyInfos[i].offsetCompressedSize); br.offset += (lastKeyInfos[i].dataCompressedSize + lastKeyInfos[i].offsetCompressedSize);
} }
} }
@ -451,7 +448,8 @@ int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *sta
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -510,25 +508,21 @@ static int32_t tsdbFileDoWriteSttBlockData(STsdbFD *fd, SBlockData *blockData, S
} }
tsdbWriterUpdVerRange(range, sttBlk->minVer, sttBlk->maxVer); tsdbWriterUpdVerRange(range, sttBlk->minVer, sttBlk->maxVer);
code = tBlockDataCompress(blockData, info, buffers, buffers + 4); TAOS_CHECK_RETURN(tBlockDataCompress(blockData, info, buffers, buffers + 4));
if (code) return code;
sttBlk->bInfo.offset = *fileSize; sttBlk->bInfo.offset = *fileSize;
sttBlk->bInfo.szKey = buffers[0].size + buffers[1].size; sttBlk->bInfo.szKey = buffers[0].size + buffers[1].size;
sttBlk->bInfo.szBlock = buffers[2].size + buffers[3].size + sttBlk->bInfo.szKey; sttBlk->bInfo.szBlock = buffers[2].size + buffers[3].size + sttBlk->bInfo.szKey;
for (int i = 0; i < 4; i++) { for (int i = 0; i < 4; i++) {
if (buffers[i].size) { if (buffers[i].size) {
code = tsdbWriteFile(fd, *fileSize, buffers[i].data, buffers[i].size, encryptAlgorithm, encryptKey); TAOS_CHECK_RETURN(tsdbWriteFile(fd, *fileSize, buffers[i].data, buffers[i].size, encryptAlgorithm, encryptKey));
if (code) return code;
*fileSize += buffers[i].size; *fileSize += buffers[i].size;
} }
} }
code = TARRAY2_APPEND_PTR(sttBlkArray, sttBlk); TAOS_CHECK_RETURN(TARRAY2_APPEND_PTR(sttBlkArray, sttBlk));
if (code) return code;
tBlockDataClear(blockData); tBlockDataClear(blockData);
return 0; return 0;
} }
@ -541,24 +535,28 @@ static int32_t tsdbSttFileDoWriteBlockData(SSttFileWriter *writer) {
tb_uid_t uid = writer->blockData->suid == 0 ? writer->blockData->uid : writer->blockData->suid; tb_uid_t uid = writer->blockData->suid == 0 ? writer->blockData->uid : writer->blockData->suid;
SColCompressInfo info = {.defaultCmprAlg = writer->config->cmprAlg, .pColCmpr = NULL}; SColCompressInfo info = {.defaultCmprAlg = writer->config->cmprAlg, .pColCmpr = NULL};
code = metaGetColCmpr(writer->config->tsdb->pVnode->pMeta, uid, &(info.pColCmpr)); code = metaGetColCmpr(writer->config->tsdb->pVnode->pMeta, uid, &(info.pColCmpr));
ASSERT(code == TSDB_CODE_SUCCESS);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileDoWriteSttBlockData(writer->fd, writer->blockData, &info, &writer->file->size, writer->sttBlkArray, TAOS_CHECK_GOTO(
writer->buffers, &writer->ctx->range, tsdbFileDoWriteSttBlockData(writer->fd, writer->blockData, &info, &writer->file->size, writer->sttBlkArray,
encryptAlgorithm, encryptKey); writer->buffers, &writer->ctx->range, encryptAlgorithm, encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
taosHashCleanup(info.pColCmpr); taosHashCleanup(info.pColCmpr);
return code; return code;
} }
static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) { static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) {
if (writer->staticBlock->numOfRecords == 0) return 0; if (writer->staticBlock->numOfRecords == 0) {
return 0;
}
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
@ -597,12 +595,10 @@ static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) {
}; };
tBufferClear(buffer0); tBufferClear(buffer0);
code = tCompressDataToBuffer(statisBlock->buffers[i].data, &info, buffer0, assist); TAOS_CHECK_GOTO(tCompressDataToBuffer(statisBlock->buffers[i].data, &info, buffer0, assist), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(
tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, info.compressedSize, encryptAlgorithm, encryptKey),
code = tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, info.compressedSize, &lino, _exit);
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
statisBlk.size[i] = info.compressedSize; statisBlk.size[i] = info.compressedSize;
statisBlk.dp->size += info.compressedSize; statisBlk.dp->size += info.compressedSize;
@ -617,44 +613,44 @@ static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) {
tBufferClear(buffer1); tBufferClear(buffer1);
for (int32_t i = 0; i < statisBlk.numOfPKs; i++) { for (int32_t i = 0; i < statisBlk.numOfPKs; i++) {
code = tValueColumnCompress(&statisBlock->firstKeyPKs[i], &compressInfo, buffer1, assist); TAOS_CHECK_GOTO(tValueColumnCompress(&statisBlock->firstKeyPKs[i], &compressInfo, buffer1, assist), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tValueColumnCompressInfoEncode(&compressInfo, buffer0), &lino, _exit);
code = tValueColumnCompressInfoEncode(&compressInfo, buffer0);
TSDB_CHECK_CODE(code, lino, _exit);
} }
for (int32_t i = 0; i < statisBlk.numOfPKs; i++) { for (int32_t i = 0; i < statisBlk.numOfPKs; i++) {
code = tValueColumnCompress(&statisBlock->lastKeyPKs[i], &compressInfo, buffer1, assist); TAOS_CHECK_GOTO(tValueColumnCompress(&statisBlock->lastKeyPKs[i], &compressInfo, buffer1, assist), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tValueColumnCompressInfoEncode(&compressInfo, buffer0), &lino, _exit);
code = tValueColumnCompressInfoEncode(&compressInfo, buffer0);
TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey); TAOS_CHECK_GOTO(
TSDB_CHECK_CODE(code, lino, _exit); tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey),
&lino, _exit);
writer->file->size += buffer0->size; writer->file->size += buffer0->size;
statisBlk.dp->size += buffer0->size; statisBlk.dp->size += buffer0->size;
code = tsdbWriteFile(writer->fd, writer->file->size, buffer1->data, buffer1->size, encryptAlgorithm, encryptKey); TAOS_CHECK_GOTO(
TSDB_CHECK_CODE(code, lino, _exit); tsdbWriteFile(writer->fd, writer->file->size, buffer1->data, buffer1->size, encryptAlgorithm, encryptKey),
&lino, _exit);
writer->file->size += buffer1->size; writer->file->size += buffer1->size;
statisBlk.dp->size += buffer1->size; statisBlk.dp->size += buffer1->size;
} }
code = TARRAY2_APPEND_PTR(writer->statisBlkArray, &statisBlk); TAOS_CHECK_GOTO(TARRAY2_APPEND_PTR(writer->statisBlkArray, &statisBlk), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
tStatisBlockClear(writer->staticBlock); tStatisBlockClear(writer->staticBlock);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
static int32_t tsdbSttFileDoWriteTombBlock(SSttFileWriter *writer) { static int32_t tsdbSttFileDoWriteTombBlock(SSttFileWriter *writer) {
if (TOMB_BLOCK_SIZE(writer->tombBlock) == 0) return 0; if (TOMB_BLOCK_SIZE(writer->tombBlock) == 0) {
return 0;
}
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
@ -662,14 +658,15 @@ static int32_t tsdbSttFileDoWriteTombBlock(SSttFileWriter *writer) {
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteTombBlock(writer->fd, writer->tombBlock, writer->config->cmprAlg, &writer->file->size, TAOS_CHECK_GOTO(
writer->tombBlkArray, writer->buffers, &writer->ctx->range, tsdbFileWriteTombBlock(writer->fd, writer->tombBlock, writer->config->cmprAlg, &writer->file->size,
encryptAlgorithm, encryptKey); writer->tombBlkArray, writer->buffers, &writer->ctx->range, encryptAlgorithm, encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -680,11 +677,8 @@ int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFData
if (ptr->size > 0) { if (ptr->size > 0) {
ptr->offset = *fileSize; ptr->offset = *fileSize;
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(sttBlkArray), ptr->size, encryptAlgorithm, TAOS_CHECK_RETURN(tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(sttBlkArray), ptr->size,
encryptKey); encryptAlgorithm, encryptKey));
if (code) {
return code;
}
*fileSize += ptr->size; *fileSize += ptr->size;
} }
@ -698,13 +692,14 @@ static int32_t tsdbSttFileDoWriteSttBlk(SSttFileWriter *writer) {
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteSttBlk(writer->fd, writer->sttBlkArray, writer->footer->sttBlkPtr, &writer->file->size, TAOS_CHECK_GOTO(tsdbFileWriteSttBlk(writer->fd, writer->sttBlkArray, writer->footer->sttBlkPtr, &writer->file->size,
encryptAlgorithm, encryptKey); encryptAlgorithm, encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -718,15 +713,16 @@ static int32_t tsdbSttFileDoWriteStatisBlk(SSttFileWriter *writer) {
writer->footer->statisBlkPtr->size = TARRAY2_DATA_LEN(writer->statisBlkArray); writer->footer->statisBlkPtr->size = TARRAY2_DATA_LEN(writer->statisBlkArray);
if (writer->footer->statisBlkPtr->size) { if (writer->footer->statisBlkPtr->size) {
writer->footer->statisBlkPtr->offset = writer->file->size; writer->footer->statisBlkPtr->offset = writer->file->size;
code = tsdbWriteFile(writer->fd, writer->file->size, (const uint8_t *)TARRAY2_DATA(writer->statisBlkArray), TAOS_CHECK_GOTO(tsdbWriteFile(writer->fd, writer->file->size, (const uint8_t *)TARRAY2_DATA(writer->statisBlkArray),
writer->footer->statisBlkPtr->size, encryptAlgorithm, encryptKey); writer->footer->statisBlkPtr->size, encryptAlgorithm, encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
writer->file->size += writer->footer->statisBlkPtr->size; writer->file->size += writer->footer->statisBlkPtr->size;
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -738,21 +734,22 @@ static int32_t tsdbSttFileDoWriteTombBlk(SSttFileWriter *writer) {
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteTombBlk(writer->fd, writer->tombBlkArray, writer->footer->tombBlkPtr, &writer->file->size, TAOS_CHECK_GOTO(tsdbFileWriteTombBlk(writer->fd, writer->tombBlkArray, writer->footer->tombBlkPtr,
encryptAlgorithm, encryptKey); &writer->file->size, encryptAlgorithm, encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm, int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm,
char *encryptKey) { char *encryptKey) {
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptAlgorithm, encryptKey); TAOS_CHECK_RETURN(
if (code) return code; tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptAlgorithm, encryptKey));
*fileSize += sizeof(*footer); *fileSize += sizeof(*footer);
return 0; return 0;
} }
@ -795,15 +792,13 @@ static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) {
char fname[TSDB_FILENAME_LEN]; char fname[TSDB_FILENAME_LEN];
tsdbTFileName(writer->config->tsdb, writer->file, fname); tsdbTFileName(writer->config->tsdb, writer->file, fname);
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd, 0); TAOS_CHECK_GOTO(tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd, 0), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
uint8_t hdr[TSDB_FHDR_SIZE] = {0}; uint8_t hdr[TSDB_FHDR_SIZE] = {0};
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbWriteFile(writer->fd, 0, hdr, sizeof(hdr), encryptAlgorithm, encryptKey); TAOS_CHECK_GOTO(tsdbWriteFile(writer->fd, 0, hdr, sizeof(hdr), encryptAlgorithm, encryptKey), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
writer->file->size += sizeof(hdr); writer->file->size += sizeof(hdr);
// range // range
@ -813,7 +808,8 @@ static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) {
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -843,34 +839,19 @@ static int32_t tsdbSttFWriterCloseCommit(SSttFileWriter *writer, TFileOpArray *o
int32_t lino; int32_t lino;
int32_t code; int32_t code;
code = tsdbSttFileDoWriteBlockData(writer); TAOS_CHECK_GOTO(tsdbSttFileDoWriteBlockData(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tsdbSttFileDoWriteStatisBlock(writer), &lino, _exit);
TAOS_CHECK_GOTO(tsdbSttFileDoWriteTombBlock(writer), &lino, _exit);
code = tsdbSttFileDoWriteStatisBlock(writer); TAOS_CHECK_GOTO(tsdbSttFileDoWriteSttBlk(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tsdbSttFileDoWriteStatisBlk(writer), &lino, _exit);
TAOS_CHECK_GOTO(tsdbSttFileDoWriteTombBlk(writer), &lino, _exit);
code = tsdbSttFileDoWriteTombBlock(writer); TAOS_CHECK_GOTO(tsdbSttFileDoWriteFooter(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tsdbSttFileDoUpdateHeader(writer), &lino, _exit);
code = tsdbSttFileDoWriteSttBlk(writer);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbSttFileDoWriteStatisBlk(writer);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbSttFileDoWriteTombBlk(writer);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbSttFileDoWriteFooter(writer);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbSttFileDoUpdateHeader(writer);
TSDB_CHECK_CODE(code, lino, _exit);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey), &lino, _exit);
tsdbCloseFile(&writer->fd); tsdbCloseFile(&writer->fd);
@ -882,12 +863,12 @@ static int32_t tsdbSttFWriterCloseCommit(SSttFileWriter *writer, TFileOpArray *o
}; };
tsdbTFileUpdVerRange(&op.nf, writer->ctx->range); tsdbTFileUpdVerRange(&op.nf, writer->ctx->range);
code = TARRAY2_APPEND(opArray, op); TAOS_CHECK_GOTO(TARRAY2_APPEND(opArray, op), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -902,7 +883,9 @@ static int32_t tsdbSttFWriterCloseAbort(SSttFileWriter *writer) {
int32_t tsdbSttFileWriterOpen(const SSttFileWriterConfig *config, SSttFileWriter **writer) { int32_t tsdbSttFileWriterOpen(const SSttFileWriterConfig *config, SSttFileWriter **writer) {
writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); writer[0] = taosMemoryCalloc(1, sizeof(*writer[0]));
if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; if (writer[0] == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
writer[0]->config[0] = config[0]; writer[0]->config[0] = config[0];
writer[0]->ctx->opened = false; writer[0]->ctx->opened = false;
@ -915,11 +898,9 @@ int32_t tsdbSttFileWriterClose(SSttFileWriter **writer, int8_t abort, TFileOpArr
if (writer[0]->ctx->opened) { if (writer[0]->ctx->opened) {
if (abort) { if (abort) {
code = tsdbSttFWriterCloseAbort(writer[0]); TAOS_CHECK_GOTO(tsdbSttFWriterCloseAbort(writer[0]), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
code = tsdbSttFWriterCloseCommit(writer[0], opArray); TAOS_CHECK_GOTO(tsdbSttFWriterCloseCommit(writer[0], opArray), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
tsdbSttFWriterDoClose(writer[0]); tsdbSttFWriterDoClose(writer[0]);
} }
@ -928,7 +909,8 @@ int32_t tsdbSttFileWriterClose(SSttFileWriter **writer, int8_t abort, TFileOpArr
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer[0]->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer[0]->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -938,20 +920,19 @@ int32_t tsdbSttFileWriteRow(SSttFileWriter *writer, SRowInfo *row) {
int32_t lino = 0; int32_t lino = 0;
if (!writer->ctx->opened) { if (!writer->ctx->opened) {
code = tsdbSttFWriterDoOpen(writer); TAOS_CHECK_GOTO(tsdbSttFWriterDoOpen(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (!TABLE_SAME_SCHEMA(row->suid, row->uid, writer->ctx->tbid->suid, writer->ctx->tbid->uid)) { if (!TABLE_SAME_SCHEMA(row->suid, row->uid, writer->ctx->tbid->suid, writer->ctx->tbid->uid)) {
code = tsdbSttFileDoWriteBlockData(writer); TAOS_CHECK_GOTO(tsdbSttFileDoWriteBlockData(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbUpdateSkmTb(writer->config->tsdb, (TABLEID *)row, writer->config->skmTb); TAOS_CHECK_GOTO(tsdbUpdateSkmTb(writer->config->tsdb, (TABLEID *)row, writer->config->skmTb), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
TABLEID id = {.suid = row->suid, .uid = row->suid ? 0 : row->uid}; TABLEID id = {
code = tBlockDataInit(writer->blockData, &id, writer->config->skmTb->pTSchema, NULL, 0); .suid = row->suid,
TSDB_CHECK_CODE(code, lino, _exit); .uid = row->suid ? 0 : row->uid,
};
TAOS_CHECK_GOTO(tBlockDataInit(writer->blockData, &id, writer->config->skmTb->pTSchema, NULL, 0), &lino, _exit);
} }
if (writer->ctx->tbid->uid != row->uid) { if (writer->ctx->tbid->uid != row->uid) {
@ -965,8 +946,7 @@ int32_t tsdbSttFileWriteRow(SSttFileWriter *writer, SRowInfo *row) {
for (;;) { for (;;) {
code = tStatisBlockPut(writer->staticBlock, row, writer->config->maxRow); code = tStatisBlockPut(writer->staticBlock, row, writer->config->maxRow);
if (code == TSDB_CODE_INVALID_PARA) { if (code == TSDB_CODE_INVALID_PARA) {
code = tsdbSttFileDoWriteStatisBlock(writer); TAOS_CHECK_GOTO(tsdbSttFileDoWriteStatisBlock(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
continue; continue;
} else { } else {
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
@ -975,9 +955,9 @@ int32_t tsdbSttFileWriteRow(SSttFileWriter *writer, SRowInfo *row) {
} }
if (row->row.type == TSDBROW_ROW_FMT) { if (row->row.type == TSDBROW_ROW_FMT) {
code = tsdbUpdateSkmRow(writer->config->tsdb, writer->ctx->tbid, // TAOS_CHECK_GOTO(tsdbUpdateSkmRow(writer->config->tsdb, writer->ctx->tbid, //
TSDBROW_SVERSION(&row->row), writer->config->skmRow); TSDBROW_SVERSION(&row->row), writer->config->skmRow),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
} }
// row to col conversion // row to col conversion
@ -989,21 +969,20 @@ int32_t tsdbSttFileWriteRow(SSttFileWriter *writer, SRowInfo *row) {
&& tsdbRowCompareWithoutVersion(&row->row, && tsdbRowCompareWithoutVersion(&row->row,
&tsdbRowFromBlockData(writer->blockData, writer->blockData->nRow - 1)) == 0 // &tsdbRowFromBlockData(writer->blockData, writer->blockData->nRow - 1)) == 0 //
) { ) {
code = tBlockDataUpdateRow(writer->blockData, &row->row, writer->config->skmRow->pTSchema); TAOS_CHECK_GOTO(tBlockDataUpdateRow(writer->blockData, &row->row, writer->config->skmRow->pTSchema), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
if (writer->blockData->nRow >= writer->config->maxRow) { if (writer->blockData->nRow >= writer->config->maxRow) {
code = tsdbSttFileDoWriteBlockData(writer); TAOS_CHECK_GOTO(tsdbSttFileDoWriteBlockData(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tBlockDataAppendRow(writer->blockData, &row->row, writer->config->skmRow->pTSchema, row->uid); TAOS_CHECK_GOTO(tBlockDataAppendRow(writer->blockData, &row->row, writer->config->skmRow->pTSchema, row->uid),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -1018,13 +997,13 @@ int32_t tsdbSttFileWriteBlockData(SSttFileWriter *writer, SBlockData *bdata) {
row->uid = bdata->uid ? bdata->uid : bdata->aUid[i]; row->uid = bdata->uid ? bdata->uid : bdata->aUid[i];
row->row = tsdbRowFromBlockData(bdata, i); row->row = tsdbRowFromBlockData(bdata, i);
code = tsdbSttFileWriteRow(writer, row); TAOS_CHECK_GOTO(tsdbSttFileWriteRow(writer, row), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -1034,31 +1013,27 @@ int32_t tsdbSttFileWriteTombRecord(SSttFileWriter *writer, const STombRecord *re
int32_t lino; int32_t lino;
if (!writer->ctx->opened) { if (!writer->ctx->opened) {
code = tsdbSttFWriterDoOpen(writer); TAOS_CHECK_GOTO(tsdbSttFWriterDoOpen(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
if (writer->blockData->nRow > 0) { if (writer->blockData->nRow > 0) {
code = tsdbSttFileDoWriteBlockData(writer); TAOS_CHECK_GOTO(tsdbSttFileDoWriteBlockData(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (STATIS_BLOCK_SIZE(writer->staticBlock) > 0) { if (STATIS_BLOCK_SIZE(writer->staticBlock) > 0) {
code = tsdbSttFileDoWriteStatisBlock(writer); TAOS_CHECK_GOTO(tsdbSttFileDoWriteStatisBlock(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
code = tTombBlockPut(writer->tombBlock, record); TAOS_CHECK_GOTO(tTombBlockPut(writer->tombBlock, record), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
if (TOMB_BLOCK_SIZE(writer->tombBlock) >= writer->config->maxRow) { if (TOMB_BLOCK_SIZE(writer->tombBlock) >= writer->config->maxRow) {
code = tsdbSttFileDoWriteTombBlock(writer); TAOS_CHECK_GOTO(tsdbSttFileDoWriteTombBlock(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} else { } else {
tsdbTrace("vgId:%d write tomb record to stt file:%s, cid:%" PRId64 ", suid:%" PRId64 ", uid:%" PRId64 tsdbTrace("vgId:%d write tomb record to stt file:%s, cid:%" PRId64 ", suid:%" PRId64 ", uid:%" PRId64
", version:%" PRId64, ", version:%" PRId64,

208
source/dnode/vnode/src/tsdb/tsdbUpgrade.c
View File

@ -58,12 +58,10 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
// read SBlockIdx array // read SBlockIdx array
if ((ctx->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx))) == NULL) { if ((ctx->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx))) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbReadBlockIdx(reader, ctx->aBlockIdx); TAOS_CHECK_GOTO(tsdbReadBlockIdx(reader, ctx->aBlockIdx), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
if (taosArrayGetSize(ctx->aBlockIdx) > 0) { if (taosArrayGetSize(ctx->aBlockIdx) > 0) {
// init/open file fd // init/open file fd
@ -77,22 +75,19 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
.maxVer = VERSION_MIN, .maxVer = VERSION_MIN,
}; };
code = tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_HEAD]); TAOS_CHECK_GOTO(tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_HEAD]), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// open fd // open fd
char fname[TSDB_FILENAME_LEN]; char fname[TSDB_FILENAME_LEN];
tsdbTFileName(tsdb, &file, fname); tsdbTFileName(tsdb, &file, fname);
code = tsdbOpenFile(fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd, 0); TAOS_CHECK_GOTO(tsdbOpenFile(fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd, 0), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// convert // convert
for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(ctx->aBlockIdx); ++iBlockIdx) { for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(ctx->aBlockIdx); ++iBlockIdx) {
SBlockIdx *pBlockIdx = taosArrayGet(ctx->aBlockIdx, iBlockIdx); SBlockIdx *pBlockIdx = taosArrayGet(ctx->aBlockIdx, iBlockIdx);
code = tsdbReadDataBlk(reader, pBlockIdx, ctx->mDataBlk); TAOS_CHECK_GOTO(tsdbReadDataBlk(reader, pBlockIdx, ctx->mDataBlk), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
for (int32_t iDataBlk = 0; iDataBlk < ctx->mDataBlk->nItem; ++iDataBlk) { for (int32_t iDataBlk = 0; iDataBlk < ctx->mDataBlk->nItem; ++iDataBlk) {
SDataBlk dataBlk[1]; SDataBlk dataBlk[1];
@ -134,12 +129,12 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
record.count = 0; record.count = 0;
} }
code = tBrinBlockPut(ctx->brinBlock, &record); TAOS_CHECK_GOTO(tBrinBlockPut(ctx->brinBlock, &record), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
if (ctx->brinBlock->numOfRecords >= ctx->maxRow) { if (ctx->brinBlock->numOfRecords >= ctx->maxRow) {
SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size, code =
tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size,
ctx->brinBlkArray, ctx->buffers, &range, ctx->encryptAlgorithm, ctx->encryptKey); ctx->brinBlkArray, ctx->buffers, &range, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
@ -148,28 +143,28 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
if (ctx->brinBlock->numOfRecords > 0) { if (ctx->brinBlock->numOfRecords > 0) {
SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size, TAOS_CHECK_GOTO(
ctx->brinBlkArray, ctx->buffers, &range, ctx->encryptAlgorithm, ctx->encryptKey); tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size,
TSDB_CHECK_CODE(code, lino, _exit); ctx->brinBlkArray, ctx->buffers, &range, ctx->encryptAlgorithm, ctx->encryptKey),
&lino, _exit);
} }
code = tsdbFileWriteBrinBlk(ctx->fd, ctx->brinBlkArray, ctx->footer->brinBlkPtr, TAOS_CHECK_GOTO(tsdbFileWriteBrinBlk(ctx->fd, ctx->brinBlkArray, ctx->footer->brinBlkPtr,
&fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->encryptAlgorithm, ctx->encryptKey); &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->encryptAlgorithm, ctx->encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
code = tsdbFileWriteHeadFooter(ctx->fd, &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->footer, ctx->encryptAlgorithm, code = tsdbFileWriteHeadFooter(ctx->fd, &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->footer, ctx->encryptAlgorithm,
ctx->encryptKey); ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey); TAOS_CHECK_GOTO(tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
tsdbCloseFile(&ctx->fd); tsdbCloseFile(&ctx->fd);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
TARRAY2_DESTROY(ctx->brinBlkArray, NULL); TARRAY2_DESTROY(ctx->brinBlkArray, NULL);
tBrinBlockDestroy(ctx->brinBlock); tBrinBlockDestroy(ctx->brinBlock);
@ -200,12 +195,11 @@ static int32_t tsdbUpgradeData(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
.maxVer = VERSION_MIN, .maxVer = VERSION_MIN,
}; };
code = tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_DATA]); TAOS_CHECK_GOTO(tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_DATA]), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -228,12 +222,11 @@ static int32_t tsdbUpgradeSma(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *r
.maxVer = VERSION_MIN, .maxVer = VERSION_MIN,
}; };
code = tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_SMA]); TAOS_CHECK_GOTO(tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_SMA]), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -245,12 +238,10 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade
SArray *aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); SArray *aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
if (aSttBlk == NULL) { if (aSttBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbReadSttBlk(reader, iStt, aSttBlk); TAOS_CHECK_GOTO(tsdbReadSttBlk(reader, iStt, aSttBlk), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
if (taosArrayGetSize(aSttBlk) > 0) { if (taosArrayGetSize(aSttBlk) > 0) {
SSttFile *pSttF = pDFileSet->aSttF[iStt]; SSttFile *pSttF = pDFileSet->aSttF[iStt];
@ -278,31 +269,27 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade
.minVer = VERSION_MAX, .minVer = VERSION_MAX,
.maxVer = VERSION_MIN, .maxVer = VERSION_MIN,
}; };
code = tsdbTFileObjInit(tsdb, &file, &fobj); TAOS_CHECK_GOTO(tsdbTFileObjInit(tsdb, &file, &fobj), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit1);
code = tsdbOpenFile(fobj->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd, 0); TAOS_CHECK_GOTO(tsdbOpenFile(fobj->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd, 0), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit1);
for (int32_t iSttBlk = 0; iSttBlk < taosArrayGetSize(aSttBlk); iSttBlk++) { for (int32_t iSttBlk = 0; iSttBlk < taosArrayGetSize(aSttBlk); iSttBlk++) {
code = TARRAY2_APPEND_PTR(ctx->sttBlkArray, (SSttBlk *)taosArrayGet(aSttBlk, iSttBlk)); TAOS_CHECK_GOTO(TARRAY2_APPEND_PTR(ctx->sttBlkArray, (SSttBlk *)taosArrayGet(aSttBlk, iSttBlk)), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit1);
} }
code = tsdbFileWriteSttBlk(ctx->fd, ctx->sttBlkArray, ctx->footer->sttBlkPtr, &fobj->f->size, ctx->encryptAlgorithm, code = tsdbFileWriteSttBlk(ctx->fd, ctx->sttBlkArray, ctx->footer->sttBlkPtr, &fobj->f->size, ctx->encryptAlgorithm,
ctx->encryptKey); ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit1); TSDB_CHECK_CODE(code, lino, _exit1);
code = tsdbFileWriteSttFooter(ctx->fd, ctx->footer, &fobj->f->size, ctx->encryptAlgorithm, ctx->encryptKey); TAOS_CHECK_GOTO(
TSDB_CHECK_CODE(code, lino, _exit1); tsdbFileWriteSttFooter(ctx->fd, ctx->footer, &fobj->f->size, ctx->encryptAlgorithm, ctx->encryptKey), &lino,
_exit);
code = tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey); TAOS_CHECK_GOTO(tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit1);
tsdbCloseFile(&ctx->fd); tsdbCloseFile(&ctx->fd);
code = TARRAY2_APPEND(lvl->fobjArr, fobj); TAOS_CHECK_GOTO(TARRAY2_APPEND(lvl->fobjArr, fobj), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit1);
_exit1: _exit1:
TARRAY2_DESTROY(ctx->sttBlkArray, NULL); TARRAY2_DESTROY(ctx->sttBlkArray, NULL);
@ -310,7 +297,7 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
taosArrayDestroy(aSttBlk); taosArrayDestroy(aSttBlk);
return code; return code;
@ -325,24 +312,21 @@ static int32_t tsdbUpgradeStt(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *r
} }
SSttLvl *lvl; SSttLvl *lvl;
code = tsdbSttLvlInit(0, &lvl); TAOS_CHECK_GOTO(tsdbSttLvlInit(0, &lvl), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
for (int32_t iStt = 0; iStt < pDFileSet->nSttF; ++iStt) { for (int32_t iStt = 0; iStt < pDFileSet->nSttF; ++iStt) {
code = tsdbUpgradeSttFile(tsdb, pDFileSet, reader, fset, iStt, lvl); TAOS_CHECK_GOTO(tsdbUpgradeSttFile(tsdb, pDFileSet, reader, fset, iStt, lvl), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (TARRAY2_SIZE(lvl->fobjArr) > 0) { if (TARRAY2_SIZE(lvl->fobjArr) > 0) {
code = TARRAY2_APPEND(fset->lvlArr, lvl); TAOS_CHECK_GOTO(TARRAY2_APPEND(fset->lvlArr, lvl), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
tsdbSttLvlClear(&lvl); tsdbSttLvlClear(&lvl);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -356,40 +340,33 @@ static int32_t tsdbUpgradeFileSet(STsdb *tsdb, SDFileSet *pDFileSet, TFileSetArr
SDataFReader *reader; SDataFReader *reader;
STFileSet *fset; STFileSet *fset;
code = tsdbTFileSetInit(pDFileSet->fid, &fset); TAOS_CHECK_GOTO(tsdbTFileSetInit(pDFileSet->fid, &fset), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbDataFReaderOpen(&reader, tsdb, pDFileSet); TAOS_CHECK_GOTO(tsdbDataFReaderOpen(&reader, tsdb, pDFileSet), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// .head // .head
code = tsdbUpgradeHead(tsdb, pDFileSet, reader, fset); TAOS_CHECK_GOTO(tsdbUpgradeHead(tsdb, pDFileSet, reader, fset), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// .data // .data
code = tsdbUpgradeData(tsdb, pDFileSet, reader, fset); TAOS_CHECK_GOTO(tsdbUpgradeData(tsdb, pDFileSet, reader, fset), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// .sma // .sma
code = tsdbUpgradeSma(tsdb, pDFileSet, reader, fset); TAOS_CHECK_GOTO(tsdbUpgradeSma(tsdb, pDFileSet, reader, fset), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// .stt // .stt
if (pDFileSet->nSttF > 0) { if (pDFileSet->nSttF > 0) {
code = tsdbUpgradeStt(tsdb, pDFileSet, reader, fset); TAOS_CHECK_GOTO(tsdbUpgradeStt(tsdb, pDFileSet, reader, fset), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
tsdbDataFReaderClose(&reader); tsdbDataFReaderClose(&reader);
code = TARRAY2_APPEND(fileSetArray, fset); TAOS_CHECK_GOTO(TARRAY2_APPEND(fileSetArray, fset), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
tsdbInfo("vgId:%d upgrade file set end, fid:%d", TD_VID(tsdb->pVnode), pDFileSet->fid); tsdbInfo("vgId:%d upgrade file set end, fid:%d", TD_VID(tsdb->pVnode), pDFileSet->fid);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -411,8 +388,7 @@ static int32_t tsdbUpgradeOpenTombFile(STsdb *tsdb, STFileSet *fset, STsdbFD **f
.maxVer = VERSION_MIN, .maxVer = VERSION_MIN,
}; };
code = tsdbTFileObjInit(tsdb, &file, fobj); TAOS_CHECK_GOTO(tsdbTFileObjInit(tsdb, &file, fobj), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
fset->farr[TSDB_FTYPE_TOMB] = *fobj; fset->farr[TSDB_FTYPE_TOMB] = *fobj;
} else { // to .stt file } else { // to .stt file
@ -429,28 +405,26 @@ static int32_t tsdbUpgradeOpenTombFile(STsdb *tsdb, STFileSet *fset, STsdbFD **f
.maxVer = VERSION_MIN, .maxVer = VERSION_MIN,
}; };
code = tsdbTFileObjInit(tsdb, &file, fobj); TAOS_CHECK_GOTO(tsdbTFileObjInit(tsdb, &file, fobj), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = TARRAY2_APPEND(lvl->fobjArr, fobj[0]); TAOS_CHECK_GOTO(TARRAY2_APPEND(lvl->fobjArr, fobj[0]), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
char fname[TSDB_FILENAME_LEN] = {0}; char fname[TSDB_FILENAME_LEN] = {0};
code = tsdbOpenFile(fobj[0]->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CREATE, fd, 0); TAOS_CHECK_GOTO(
TSDB_CHECK_CODE(code, lino, _exit); tsdbOpenFile(fobj[0]->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CREATE, fd, 0), &lino,
_exit);
uint8_t hdr[TSDB_FHDR_SIZE] = {0}; uint8_t hdr[TSDB_FHDR_SIZE] = {0};
int32_t encryptAlgorithm = tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbWriteFile(fd[0], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey); TAOS_CHECK_GOTO(tsdbWriteFile(fd[0], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
fobj[0]->f->size += TSDB_FHDR_SIZE; fobj[0]->f->size += TSDB_FHDR_SIZE;
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -488,15 +462,13 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *
tsdbFidKeyRange(fset->fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &ctx->minKey, &ctx->maxKey); tsdbFidKeyRange(fset->fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &ctx->minKey, &ctx->maxKey);
if ((ctx->aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) { if ((ctx->aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) {
SDelIdx *pDelIdx = (SDelIdx *)taosArrayGet(aDelIdx, iDelIdx); SDelIdx *pDelIdx = (SDelIdx *)taosArrayGet(aDelIdx, iDelIdx);
code = tsdbReadDelData(reader, pDelIdx, ctx->aDelData); TAOS_CHECK_GOTO(tsdbReadDelData(reader, pDelIdx, ctx->aDelData), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
for (int32_t iDelData = 0; iDelData < taosArrayGetSize(ctx->aDelData); iDelData++) { for (int32_t iDelData = 0; iDelData < taosArrayGetSize(ctx->aDelData); iDelData++) {
SDelData *pDelData = (SDelData *)taosArrayGet(ctx->aDelData, iDelData); SDelData *pDelData = (SDelData *)taosArrayGet(ctx->aDelData, iDelData);
@ -509,31 +481,30 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *
.ekey = pDelData->eKey, .ekey = pDelData->eKey,
}; };
code = tTombBlockPut(ctx->tombBlock, &record); TAOS_CHECK_GOTO(tTombBlockPut(ctx->tombBlock, &record), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
if (TOMB_BLOCK_SIZE(ctx->tombBlock) > ctx->maxRow) { if (TOMB_BLOCK_SIZE(ctx->tombBlock) > ctx->maxRow) {
if (ctx->fd == NULL) { if (ctx->fd == NULL) {
code = tsdbUpgradeOpenTombFile(tsdb, fset, &ctx->fd, &ctx->fobj, &ctx->toStt); TAOS_CHECK_GOTO(tsdbUpgradeOpenTombFile(tsdb, fset, &ctx->fd, &ctx->fobj, &ctx->toStt), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray, TAOS_CHECK_GOTO(
ctx->buffers, &tombRange, ctx->encryptAlgorithm, ctx->encryptKey); tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray,
TSDB_CHECK_CODE(code, lino, _exit); ctx->buffers, &tombRange, ctx->encryptAlgorithm, ctx->encryptKey),
&lino, _exit);
} }
} }
} }
if (TOMB_BLOCK_SIZE(ctx->tombBlock) > 0) { if (TOMB_BLOCK_SIZE(ctx->tombBlock) > 0) {
if (ctx->fd == NULL) { if (ctx->fd == NULL) {
code = tsdbUpgradeOpenTombFile(tsdb, fset, &ctx->fd, &ctx->fobj, &ctx->toStt); TAOS_CHECK_GOTO(tsdbUpgradeOpenTombFile(tsdb, fset, &ctx->fd, &ctx->fobj, &ctx->toStt), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray, TAOS_CHECK_GOTO(
ctx->buffers, &tombRange, ctx->encryptAlgorithm, ctx->encryptKey); tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray,
TSDB_CHECK_CODE(code, lino, _exit); ctx->buffers, &tombRange, ctx->encryptAlgorithm, ctx->encryptKey),
&lino, _exit);
} }
if (ctx->fd != NULL) { if (ctx->fd != NULL) {
@ -542,8 +513,8 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *
ctx->encryptAlgorithm, ctx->encryptKey); ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFileWriteSttFooter(ctx->fd, ctx->sttFooter, &ctx->fobj->f->size, ctx->encryptAlgorithm, code =
ctx->encryptKey); tsdbFileWriteSttFooter(ctx->fd, ctx->sttFooter, &ctx->fobj->f->size, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->tombFooter->tombBlkPtr, &ctx->fobj->f->size, code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->tombFooter->tombBlkPtr, &ctx->fobj->f->size,
@ -555,15 +526,14 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey); TAOS_CHECK_GOTO(tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
tsdbCloseFile(&ctx->fd); tsdbCloseFile(&ctx->fd);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
for (int32_t i = 0; i < ARRAY_SIZE(ctx->buffers); i++) { for (int32_t i = 0; i < ARRAY_SIZE(ctx->buffers); i++) {
tBufferDestroy(ctx->buffers + i); tBufferDestroy(ctx->buffers + i);
@ -582,27 +552,23 @@ static int32_t tsdbUpgradeTombFile(STsdb *tsdb, SDelFile *pDelFile, TFileSetArra
SArray *aDelIdx = NULL; SArray *aDelIdx = NULL;
if ((aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) { if ((aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbDelFReaderOpen(&reader, pDelFile, tsdb); TAOS_CHECK_GOTO(tsdbDelFReaderOpen(&reader, pDelFile, tsdb), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbReadDelIdx(reader, aDelIdx); TAOS_CHECK_GOTO(tsdbReadDelIdx(reader, aDelIdx), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
if (taosArrayGetSize(aDelIdx) > 0) { if (taosArrayGetSize(aDelIdx) > 0) {
STFileSet *fset; STFileSet *fset;
TARRAY2_FOREACH(fileSetArray, fset) { TARRAY2_FOREACH(fileSetArray, fset) {
code = tsdbDumpTombDataToFSet(tsdb, reader, aDelIdx, fset); TAOS_CHECK_GOTO(tsdbDumpTombDataToFSet(tsdb, reader, aDelIdx, fset), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
tsdbDelFReaderClose(&reader); tsdbDelFReaderClose(&reader);
taosArrayDestroy(aDelIdx); taosArrayDestroy(aDelIdx);
@ -615,19 +581,17 @@ static int32_t tsdbDoUpgradeFileSystem(STsdb *tsdb, TFileSetArray *fileSetArray)
// upgrade each file set // upgrade each file set
for (int32_t i = 0; i < taosArrayGetSize(tsdb->fs.aDFileSet); i++) { for (int32_t i = 0; i < taosArrayGetSize(tsdb->fs.aDFileSet); i++) {
code = tsdbUpgradeFileSet(tsdb, taosArrayGet(tsdb->fs.aDFileSet, i), fileSetArray); TAOS_CHECK_GOTO(tsdbUpgradeFileSet(tsdb, taosArrayGet(tsdb->fs.aDFileSet, i), fileSetArray), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
// upgrade tomb file // upgrade tomb file
if (tsdb->fs.pDelFile != NULL) { if (tsdb->fs.pDelFile != NULL) {
code = tsdbUpgradeTombFile(tsdb, tsdb->fs.pDelFile, fileSetArray); TAOS_CHECK_GOTO(tsdbUpgradeTombFile(tsdb, tsdb->fs.pDelFile, fileSetArray), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -639,24 +603,21 @@ static int32_t tsdbUpgradeFileSystem(STsdb *tsdb, int8_t rollback) {
TFileSetArray fileSetArray[1] = {0}; TFileSetArray fileSetArray[1] = {0};
// open old file system // open old file system
code = tsdbFSOpen(tsdb, rollback); TAOS_CHECK_GOTO(tsdbFSOpen(tsdb, rollback), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbDoUpgradeFileSystem(tsdb, fileSetArray); TAOS_CHECK_GOTO(tsdbDoUpgradeFileSystem(tsdb, fileSetArray), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// close file system // close file system
code = tsdbFSClose(tsdb); TAOS_CHECK_GOTO(tsdbFSClose(tsdb), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// save new file system // save new file system
char fname[TSDB_FILENAME_LEN]; char fname[TSDB_FILENAME_LEN];
current_fname(tsdb, fname, TSDB_FCURRENT); current_fname(tsdb, fname, TSDB_FCURRENT);
code = save_fs(fileSetArray, fname); TAOS_CHECK_GOTO(save_fs(fileSetArray, fname), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
} }
TARRAY2_DESTROY(fileSetArray, tsdbTFileSetClear); TARRAY2_DESTROY(fileSetArray, tsdbTFileSetClear);
@ -669,9 +630,8 @@ int32_t tsdbCheckAndUpgradeFileSystem(STsdb *tsdb, int8_t rollback) {
tsdbGetCurrentFName(tsdb, fname, NULL); tsdbGetCurrentFName(tsdb, fname, NULL);
if (!taosCheckExistFile(fname)) return 0; if (!taosCheckExistFile(fname)) return 0;
int32_t code = tsdbUpgradeFileSystem(tsdb, rollback); TAOS_CHECK_RETURN(tsdbUpgradeFileSystem(tsdb, rollback));
if (code) return code;
taosRemoveFile(fname); (void)taosRemoveFile(fname);
return 0; return 0;
} }

34
source/dnode/vnode/src/tsdb/tsdbWrite.c
View File

@ -28,6 +28,7 @@ int64_t tsMaxKeyByPrecision[] = {31556995200000L, 31556995200000000L, 9214646400
// static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); // static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg);
int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq2 *pMsg, SSubmitRsp2 *pRsp) { int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq2 *pMsg, SSubmitRsp2 *pRsp) {
int32_t code;
int32_t arrSize = 0; int32_t arrSize = 0;
int32_t affectedrows = 0; int32_t affectedrows = 0;
int32_t numOfRows = 0; int32_t numOfRows = 0;
@ -39,18 +40,16 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq2 *pMsg, SSubmitRsp2
arrSize = taosArrayGetSize(pMsg->aSubmitTbData); arrSize = taosArrayGetSize(pMsg->aSubmitTbData);
// scan and convert // scan and convert
if ((terrno = tsdbScanAndConvertSubmitMsg(pTsdb, pMsg)) < 0) { if ((code = tsdbScanAndConvertSubmitMsg(pTsdb, pMsg)) < 0) {
if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) { if (code != TSDB_CODE_TDB_TABLE_RECONFIGURE) {
tsdbError("vgId:%d, failed to insert data since %s", TD_VID(pTsdb->pVnode), tstrerror(terrno)); tsdbError("vgId:%d, failed to insert data since %s", TD_VID(pTsdb->pVnode), tstrerror(terrno));
} }
return -1; return code;
} }
// loop to insert // loop to insert
for (int32_t i = 0; i < arrSize; ++i) { for (int32_t i = 0; i < arrSize; ++i) {
if ((terrno = tsdbInsertTableData(pTsdb, version, taosArrayGet(pMsg->aSubmitTbData, i), &affectedrows)) < 0) { TAOS_CHECK_RETURN(tsdbInsertTableData(pTsdb, version, taosArrayGet(pMsg->aSubmitTbData, i), &affectedrows));
return -1;
}
} }
if (pRsp != NULL) { if (pRsp != NULL) {
@ -74,22 +73,12 @@ static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, TSKEY rowK
} }
int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq2 *pMsg) { int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq2 *pMsg) {
int32_t code = 0;
STsdbKeepCfg *pCfg = &pTsdb->keepCfg; STsdbKeepCfg *pCfg = &pTsdb->keepCfg;
TSKEY now = taosGetTimestamp(pCfg->precision); TSKEY now = taosGetTimestamp(pCfg->precision);
TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2; TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2;
TSKEY maxKey = tsMaxKeyByPrecision[pCfg->precision]; TSKEY maxKey = tsMaxKeyByPrecision[pCfg->precision];
int32_t size = taosArrayGetSize(pMsg->aSubmitTbData); int32_t size = taosArrayGetSize(pMsg->aSubmitTbData);
/*
int32_t nlevel = tfsGetLevel(pTsdb->pVnode->pTfs);
if (nlevel > 1 && tsS3Enabled) {
if (nlevel == 3) {
minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep1;
} else if (nlevel == 2) {
minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep0;
}
}
*/
for (int32_t i = 0; i < size; ++i) { for (int32_t i = 0; i < size; ++i) {
SSubmitTbData *pData = TARRAY_GET_ELEM(pMsg->aSubmitTbData, i); SSubmitTbData *pData = TARRAY_GET_ELEM(pMsg->aSubmitTbData, i);
if (pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { if (pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) {
@ -99,22 +88,17 @@ int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq2 *pMsg) {
int32_t nRows = aColData[0].nVal; int32_t nRows = aColData[0].nVal;
TSKEY *aKey = (TSKEY *)aColData[0].pData; TSKEY *aKey = (TSKEY *)aColData[0].pData;
for (int32_t r = 0; r < nRows; ++r) { for (int32_t r = 0; r < nRows; ++r) {
if ((code = tsdbCheckRowRange(pTsdb, pData->uid, aKey[r], minKey, maxKey, now)) < 0) { TAOS_CHECK_RETURN(tsdbCheckRowRange(pTsdb, pData->uid, aKey[r], minKey, maxKey, now));
goto _exit;
}
} }
} }
} else { } else {
int32_t nRows = taosArrayGetSize(pData->aRowP); int32_t nRows = taosArrayGetSize(pData->aRowP);
for (int32_t r = 0; r < nRows; ++r) { for (int32_t r = 0; r < nRows; ++r) {
SRow *pRow = (SRow *)taosArrayGetP(pData->aRowP, r); SRow *pRow = (SRow *)taosArrayGetP(pData->aRowP, r);
if ((code = tsdbCheckRowRange(pTsdb, pData->uid, pRow->ts, minKey, maxKey, now)) < 0) { TAOS_CHECK_RETURN(tsdbCheckRowRange(pTsdb, pData->uid, pRow->ts, minKey, maxKey, now));
goto _exit;
}
} }
} }
} }
_exit: return 0;
return code;
} }