From f45d10bb282ca7f506e7ad08b8e48db47a5e40da Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 May 2023 18:15:31 +0800 Subject: [PATCH] more code --- .../dnode/vnode/src/tsdb/dev/inc/tsdbFSet.h | 1 + .../dnode/vnode/src/tsdb/dev/inc/tsdbFile.h | 2 +- source/dnode/vnode/src/tsdb/dev/tsdbFS.c | 101 +++++++++++++- source/dnode/vnode/src/tsdb/dev/tsdbFSet.c | 32 +++++ source/dnode/vnode/src/tsdb/dev/tsdbFile.c | 125 ++++++++++++++++-- 5 files changed, 246 insertions(+), 15 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/dev/inc/tsdbFSet.h b/source/dnode/vnode/src/tsdb/dev/inc/tsdbFSet.h index 272235c2df..8fbab80df7 100644 --- a/source/dnode/vnode/src/tsdb/dev/inc/tsdbFSet.h +++ b/source/dnode/vnode/src/tsdb/dev/inc/tsdbFSet.h @@ -34,6 +34,7 @@ typedef enum { } tsdb_fop_t; int32_t tsdbFileSetToJson(const STFileSet *fset, cJSON *json); +int32_t tsdbFileSetFromJson(const cJSON *json, STFileSet *fset); int32_t tsdbFileSetCreate(int32_t fid, STFileSet **ppSet); int32_t tsdbFileSetEdit(STFileSet *pSet, SFileOp *pOp); diff --git a/source/dnode/vnode/src/tsdb/dev/inc/tsdbFile.h b/source/dnode/vnode/src/tsdb/dev/inc/tsdbFile.h index 2d8d4b15d8..70fbb752af 100644 --- a/source/dnode/vnode/src/tsdb/dev/inc/tsdbFile.h +++ b/source/dnode/vnode/src/tsdb/dev/inc/tsdbFile.h @@ -36,6 +36,7 @@ typedef enum { #define TSDB_FTYPE_MAX (TSDB_FTYPE_TOMB + 1) int32_t tsdbTFileToJson(const STFile *f, cJSON *json); +int32_t tsdbTFileFromJson(const cJSON *json, tsdb_ftype_t ftype, STFile **f); int32_t tsdbTFileInit(STsdb *pTsdb, STFile *pFile); int32_t tsdbTFileClear(STFile *pFile); @@ -44,7 +45,6 @@ struct STFile { LISTD(STFile) listNode; char fname[TSDB_FILENAME_LEN]; int32_t ref; - int32_t state; tsdb_ftype_t type; SDiskID did; diff --git a/source/dnode/vnode/src/tsdb/dev/tsdbFS.c b/source/dnode/vnode/src/tsdb/dev/tsdbFS.c index 47d00672d2..f1ee05ce48 100644 --- a/source/dnode/vnode/src/tsdb/dev/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/dev/tsdbFS.c @@ -102,7 +102,7 @@ _exit: } static int32_t save_json(const cJSON *json, const char *fname) { - int32_t code; + int32_t code = 0; char *data = cJSON_Print(json); if (data == NULL) return TSDB_CODE_OUT_OF_MEMORY; @@ -130,6 +130,43 @@ _exit: return code; } +static int32_t load_json(const char *fname, cJSON **json) { + int32_t code = 0; + void *data = NULL; + + TdFilePtr fp = taosOpenFile(fname, TD_FILE_READ); + if (fp == NULL) return TAOS_SYSTEM_ERROR(code); + + int64_t size; + if (taosFStatFile(fp, &size, NULL) < 0) { + code = TAOS_SYSTEM_ERROR(code); + goto _exit; + } + + data = taosMemoryMalloc(size); + if (data == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + if (taosReadFile(fp, data, size) < 0) { + code = TAOS_SYSTEM_ERROR(code); + goto _exit; + } + + json[0] = cJSON_Parse(data); + if (json[0] == NULL) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } + +_exit: + taosCloseFile(&fp); + if (data) taosMemoryFree(data); + if (code) json[0] = NULL; + return code; +} + static int32_t save_fs(int64_t eid, SArray *aTFileSet, const char *fname) { int32_t code = 0; int32_t lino = 0; @@ -181,9 +218,63 @@ _exit: return code; } -static int32_t load_fs(const char *fname, STFileSystem *pFS) { - ASSERTS(0, "TODO: Not implemented yet"); - return 0; +static int32_t load_fs(const char *fname, SArray *aTFileSet, int64_t *eid) { + int32_t code = 0; + int32_t lino = 0; + + taosArrayClear(aTFileSet); + + // load json + cJSON *json = NULL; + code = load_json(fname, &json); + TSDB_CHECK_CODE(code, lino, _exit) + + // parse json + const cJSON *item; + + /* fmtv */ + item = cJSON_GetObjectItem(json, "fmtv"); + if (cJSON_IsNumber(item)) { + ASSERT(item->valuedouble == 1); + } else { + code = TSDB_CODE_FILE_CORRUPTED; + TSDB_CHECK_CODE(code, lino, _exit) + } + + /* eid */ + item = cJSON_GetObjectItem(json, "eid"); + if (cJSON_IsNumber(item)) { + eid[0] = item->valuedouble; + } else { + code = TSDB_CODE_FILE_CORRUPTED; + TSDB_CHECK_CODE(code, lino, _exit) + } + + /* fset */ + item = cJSON_GetObjectItem(json, "fset"); + if (cJSON_IsArray(item)) { + const cJSON *titem; + cJSON_ArrayForEach(titem, item) { + STFileSet *pFileSet = taosArrayReserve(aTFileSet, 1); + if (pFileSet == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbFileSetFromJson(titem, pFileSet); + TSDB_CHECK_CODE(code, lino, _exit) + } + } else { + code = TSDB_CODE_FILE_CORRUPTED; + TSDB_CHECK_CODE(code, lino, _exit) + } + +_exit: + if (code) { + tsdbError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname); + } + if (json) cJSON_Delete(json); + return code; } static int32_t commit_edit(STFileSystem *pFS, tsdb_fs_edit_t etype) { @@ -249,7 +340,7 @@ static int32_t open_fs(STFileSystem *pFS, int8_t rollback) { current_fname(pTsdb, mCurrent, TSDB_FCURRENT_M); if (taosCheckExistFile(fCurrent)) { // current.json exists - code = load_fs(fCurrent, pFS); + code = load_fs(fCurrent, pFS->cstate, &pFS->nextEditId); TSDB_CHECK_CODE(code, lino, _exit); // check current.json.commit existence diff --git a/source/dnode/vnode/src/tsdb/dev/tsdbFSet.c b/source/dnode/vnode/src/tsdb/dev/tsdbFSet.c index d6af8e8877..93f862cf12 100644 --- a/source/dnode/vnode/src/tsdb/dev/tsdbFSet.c +++ b/source/dnode/vnode/src/tsdb/dev/tsdbFSet.c @@ -38,6 +38,11 @@ static int32_t stt_lvl_to_json(const SSttLvl *lvl, cJSON *json) { return 0; } +static int32_t stt_lvl_from_json(const cJSON *json, SSttLvl *lvl) { + // TODO + return 0; +} + int32_t tsdbFileSetCreate(int32_t fid, struct STFileSet **ppSet) { int32_t code = 0; @@ -94,6 +99,33 @@ int32_t tsdbFileSetToJson(const STFileSet *fset, cJSON *json) { return 0; } +int32_t tsdbFileSetFromJson(const cJSON *json, STFileSet *fset) { + const cJSON *item; + + /* fid */ + item = cJSON_GetObjectItem(json, "fid"); + if (cJSON_IsNumber(item)) { + fset->fid = item->valueint; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + int32_t code = tsdbTFileFromJson(json, ftype, &fset->farr[ftype]); + if (code) return code; + } + + // each level + item = cJSON_GetObjectItem(json, "stt"); + if (cJSON_IsArray(item)) { + // TODO + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + return 0; +} + int32_t tsdbEditFileSet(struct STFileSet *pFileSet, const struct SFileOp *pOp) { int32_t code = 0; ASSERTS(0, "TODO: Not implemented yet"); diff --git a/source/dnode/vnode/src/tsdb/dev/tsdbFile.c b/source/dnode/vnode/src/tsdb/dev/tsdbFile.c index 58ad4b1f93..cae89e736e 100644 --- a/source/dnode/vnode/src/tsdb/dev/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/dev/tsdbFile.c @@ -15,40 +15,54 @@ #include "inc/tsdbFile.h" +// to_json static int32_t head_to_json(const STFile *file, cJSON *json); static int32_t data_to_json(const STFile *file, cJSON *json); static int32_t sma_to_json(const STFile *file, cJSON *json); static int32_t tomb_to_json(const STFile *file, cJSON *json); static int32_t stt_to_json(const STFile *file, cJSON *json); +// from_json +static int32_t head_from_json(const cJSON *json, STFile *file); +static int32_t data_from_json(const cJSON *json, STFile *file); +static int32_t sma_from_json(const cJSON *json, STFile *file); +static int32_t tomb_from_json(const cJSON *json, STFile *file); +static int32_t stt_from_json(const cJSON *json, STFile *file); + static const struct { const char *suffix; int32_t (*to_json)(const STFile *file, cJSON *json); + int32_t (*from_json)(const cJSON *json, STFile *file); } g_tfile_info[] = { - [TSDB_FTYPE_HEAD] = {"head", head_to_json}, // - [TSDB_FTYPE_DATA] = {"data", data_to_json}, // - [TSDB_FTYPE_SMA] = {"sma", sma_to_json}, // - [TSDB_FTYPE_TOMB] = {"tomb", tomb_to_json}, // - [TSDB_FTYPE_STT] = {"stt", stt_to_json}, + [TSDB_FTYPE_HEAD] = {"head", head_to_json, head_from_json}, + [TSDB_FTYPE_DATA] = {"data", data_to_json, data_from_json}, + [TSDB_FTYPE_SMA] = {"sma", sma_to_json, sma_from_json}, + [TSDB_FTYPE_TOMB] = {"tomb", tomb_to_json, tomb_from_json}, + [TSDB_FTYPE_STT] = {"stt", stt_to_json, stt_from_json}, }; static int32_t tfile_to_json(const STFile *file, cJSON *json) { + /* did.level */ if (cJSON_AddNumberToObject(json, "did.level", file->did.level) == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } + /* did.id */ if (cJSON_AddNumberToObject(json, "did.id", file->did.id) == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } + /* fid */ if (cJSON_AddNumberToObject(json, "fid", file->fid) == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } + /* cid */ if (cJSON_AddNumberToObject(json, "cid", file->cid) == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } + /* size */ if (cJSON_AddNumberToObject(json, "size", file->size) == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -56,22 +70,66 @@ static int32_t tfile_to_json(const STFile *file, cJSON *json) { return 0; } +static int32_t tfile_from_json(const cJSON *json, STFile *file) { + const cJSON *item; + + /* did.level */ + item = cJSON_GetObjectItem(json, "did.level"); + if (cJSON_IsNumber(item)) { + file->did.level = item->valuedouble; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + /* did.id */ + item = cJSON_GetObjectItem(json, "did.id"); + if (cJSON_IsNumber(item)) { + file->did.id = item->valuedouble; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + /* fid */ + item = cJSON_GetObjectItem(json, "fid"); + if (cJSON_IsNumber(item)) { + file->fid = item->valuedouble; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + /* cid */ + item = cJSON_GetObjectItem(json, "cid"); + if (cJSON_IsNumber(item)) { + file->cid = item->valuedouble; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + /* size */ + item = cJSON_GetObjectItem(json, "size"); + if (cJSON_IsNumber(item)) { + file->size = item->valuedouble; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + return 0; +} + static int32_t head_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); } - static int32_t data_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); } - static int32_t sma_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); } - static int32_t tomb_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); } - static int32_t stt_to_json(const STFile *file, cJSON *json) { int32_t code = tfile_to_json(file, json); if (code) return code; + /* lvl */ if (cJSON_AddNumberToObject(json, "lvl", file->stt.lvl) == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } + /* nseg */ if (cJSON_AddNumberToObject(json, "nseg", file->stt.nseg) == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -79,6 +137,35 @@ static int32_t stt_to_json(const STFile *file, cJSON *json) { return 0; } +static int32_t head_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); } +static int32_t data_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); } +static int32_t sma_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); } +static int32_t tomb_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); } +static int32_t stt_from_json(const cJSON *json, STFile *file) { + int32_t code = tfile_from_json(json, file); + if (code) return code; + + const cJSON *item; + + /* lvl */ + item = cJSON_GetObjectItem(json, "lvl"); + if (cJSON_IsNumber(item)) { + file->stt.lvl = item->valuedouble; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + /* nseg */ + item = cJSON_GetObjectItem(json, "nseg"); + if (cJSON_IsNumber(item)) { + file->stt.nseg = item->valuedouble; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + return 0; +} + int32_t tsdbTFileInit(STsdb *pTsdb, STFile *pFile) { SVnode *pVnode = pTsdb->pVnode; STfs *pTfs = pVnode->pTfs; @@ -120,4 +207,24 @@ int32_t tsdbTFileToJson(const STFile *file, cJSON *json) { if (tjson == NULL) return TSDB_CODE_OUT_OF_MEMORY; return g_tfile_info[file->type].to_json(file, tjson); +} + +int32_t tsdbTFileFromJson(const cJSON *json, tsdb_ftype_t ftype, STFile **f) { + const cJSON *item = cJSON_GetObjectItem(json, g_tfile_info[ftype].suffix); + if (cJSON_IsObject(item)) { + f[0] = (STFile *)taosMemoryMalloc(sizeof(*f[0])); + if (f[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + int32_t code = g_tfile_info[ftype].from_json(item, f[0]); + if (code) { + taosMemoryFree(f[0]); + f[0] = NULL; + return code; + } + tsdbTFileInit(NULL /* TODO */, f[0]); + } else { + f[0] = NULL; + } + + return 0; } \ No newline at end of file