From 9e3e40ad6e237636f57ca6a652b770b6a929f326 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 6 Jan 2022 08:15:09 +0000 Subject: [PATCH] integrate TFS --- include/libs/tfs/tfs.h | 103 ++++++++++++++++++++++++++ include/os/os.h | 2 + source/libs/CMakeLists.txt | 1 + source/libs/tfs/CMakeLists.txt | 9 +++ {src => source/libs}/tfs/inc/tfsint.h | 0 {src => source/libs}/tfs/src/tdisk.c | 0 {src => source/libs}/tfs/src/tfs.c | 16 ++-- {src => source/libs}/tfs/src/ttier.c | 0 src/tfs/CMakeLists.txt | 12 --- 9 files changed, 122 insertions(+), 21 deletions(-) create mode 100644 include/libs/tfs/tfs.h create mode 100644 source/libs/tfs/CMakeLists.txt rename {src => source/libs}/tfs/inc/tfsint.h (100%) rename {src => source/libs}/tfs/src/tdisk.c (100%) rename {src => source/libs}/tfs/src/tfs.c (97%) rename {src => source/libs}/tfs/src/ttier.c (100%) delete mode 100644 src/tfs/CMakeLists.txt diff --git a/include/libs/tfs/tfs.h b/include/libs/tfs/tfs.h new file mode 100644 index 0000000000..11e33a3af7 --- /dev/null +++ b/include/libs/tfs/tfs.h @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TD_TFS_H +#define TD_TFS_H + +#include "tglobal.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + int level; + int id; +} SDiskID; + +#define TFS_UNDECIDED_LEVEL -1 +#define TFS_UNDECIDED_ID -1 +#define TFS_PRIMARY_LEVEL 0 +#define TFS_PRIMARY_ID 0 +#define TFS_MIN_LEVEL 0 +#define TFS_MAX_LEVEL (TSDB_MAX_TIERS - 1) + +// FS APIs ==================================== +typedef struct { + int64_t tsize; + int64_t used; + int64_t avail; +} SFSMeta; + +typedef struct { + int64_t size; + int64_t used; + int64_t free; + int16_t nAvailDisks; // # of Available disks +} STierMeta; + +int tfsInit(SDiskCfg *pDiskCfg, int ndisk); +void tfsDestroy(); +void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numLevels); +void tfsGetMeta(SFSMeta *pMeta); +void tfsAllocDisk(int expLevel, int *level, int *id); + +const char *TFS_PRIMARY_PATH(); +const char *TFS_DISK_PATH(int level, int id); + +// TFILE APIs ==================================== +typedef struct { + int level; + int id; + char rname[TSDB_FILENAME_LEN]; // REL name + char aname[TSDB_FILENAME_LEN]; // ABS name +} TFILE; + +#define TFILE_LEVEL(pf) ((pf)->level) +#define TFILE_ID(pf) ((pf)->id) +#define TFILE_NAME(pf) ((pf)->aname) +#define TFILE_REL_NAME(pf) ((pf)->rname) + +#define tfsopen(pf, flags) open(TFILE_NAME(pf), flags) +#define tfsclose(fd) close(fd) +#define tfsremove(pf) remove(TFILE_NAME(pf)) +#define tfscopy(sf, df) taosCopy(TFILE_NAME(sf), TFILE_NAME(df)) +#define tfsrename(sf, df) taosRename(TFILE_NAME(sf), TFILE_NAME(df)) + +void tfsInitFile(TFILE *pf, int level, int id, const char *bname); +bool tfsIsSameFile(const TFILE *pf1, const TFILE *pf2); +int tfsEncodeFile(void **buf, TFILE *pf); +void *tfsDecodeFile(void *buf, TFILE *pf); +void tfsbasename(const TFILE *pf, char *dest); +void tfsdirname(const TFILE *pf, char *dest); + +// DIR APIs ==================================== +int tfsMkdirAt(const char *rname, int level, int id); +int tfsMkdirRecurAt(const char *rname, int level, int id); +int tfsMkdir(const char *rname); +int tfsRmdir(const char *rname); +int tfsRename(char *orname, char *nrname); + +typedef struct TDIR TDIR; + +TDIR * tfsOpendir(const char *rname); +const TFILE *tfsReaddir(TDIR *tdir); +void tfsClosedir(TDIR *tdir); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/os/os.h b/include/os/os.h index 972880da9c..16ba191c26 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -44,6 +44,8 @@ extern "C" { #include #include #include +#include +#include #include diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 027532bbb1..1dc16c74f7 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -12,3 +12,4 @@ add_subdirectory(planner) add_subdirectory(function) add_subdirectory(qcom) add_subdirectory(qworker) +add_subdirectory(tfs) diff --git a/source/libs/tfs/CMakeLists.txt b/source/libs/tfs/CMakeLists.txt new file mode 100644 index 0000000000..1b6f662507 --- /dev/null +++ b/source/libs/tfs/CMakeLists.txt @@ -0,0 +1,9 @@ +aux_source_directory(src TFS_SRC) +add_library(tfs STATIC ${TFS_SRC}) +target_include_directories( + tfs + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/tfs" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries(tfs os util common) \ No newline at end of file diff --git a/src/tfs/inc/tfsint.h b/source/libs/tfs/inc/tfsint.h similarity index 100% rename from src/tfs/inc/tfsint.h rename to source/libs/tfs/inc/tfsint.h diff --git a/src/tfs/src/tdisk.c b/source/libs/tfs/src/tdisk.c similarity index 100% rename from src/tfs/src/tdisk.c rename to source/libs/tfs/src/tdisk.c diff --git a/src/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c similarity index 97% rename from src/tfs/src/tfs.c rename to source/libs/tfs/src/tfs.c index 547f862c20..b7c7408a23 100644 --- a/src/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -15,11 +15,11 @@ #include "os.h" -#include "hash.h" #include "taosdef.h" #include "taoserror.h" #include "tfs.h" #include "tfsint.h" +#include "thash.h" #define TMPNAME_LEN (TSDB_FILENAME_LEN * 2 + 32) @@ -270,7 +270,8 @@ int tfsMkdirRecurAt(const char *rname, int level, int id) { // Some platform may modify the contents of the string passed into dirname(). Others may return a pointer to // internal static storage space that will be overwritten by next call. For case like that, we should not use // the pointer directly in this recursion. - // See https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/dirname.3.html + // See + // https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/dirname.3.html char *dir = strdup(dirname(s)); if (tfsMkdirRecurAt(dir, level, id) < 0) { @@ -504,7 +505,6 @@ static int tfsFormatDir(char *idir, char *odir) { wordfree(&wep); return 0; - } static int tfsCheck() { @@ -599,12 +599,10 @@ void taosGetDisk() { SysDiskSize diskSize; SFSMeta fsMeta; - if (tscEmbedded) { - tfsUpdateInfo(&fsMeta, NULL, 0); - tsTotalDataDirGB = (float)(fsMeta.tsize / unit); - tsUsedDataDirGB = (float)(fsMeta.used / unit); - tsAvailDataDirGB = (float)(fsMeta.avail / unit); - } + tfsUpdateInfo(&fsMeta, NULL, 0); + tsTotalDataDirGB = (float)(fsMeta.tsize / unit); + tsUsedDataDirGB = (float)(fsMeta.used / unit); + tsAvailDataDirGB = (float)(fsMeta.avail / unit); if (taosGetDiskSize(tsLogDir, &diskSize) == 0) { tsTotalLogDirGB = (float)(diskSize.tsize / unit); diff --git a/src/tfs/src/ttier.c b/source/libs/tfs/src/ttier.c similarity index 100% rename from src/tfs/src/ttier.c rename to source/libs/tfs/src/ttier.c diff --git a/src/tfs/CMakeLists.txt b/src/tfs/CMakeLists.txt deleted file mode 100644 index 7f956f07a2..0000000000 --- a/src/tfs/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(inc) -AUX_SOURCE_DIRECTORY(src SRC) -ADD_LIBRARY(tfs ${SRC}) -TARGET_LINK_LIBRARIES(tfs tutil) - -IF (TD_LINUX) - # Someone has no gtest directory, so comment it - # ADD_SUBDIRECTORY(tests) -ENDIF ()