xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #9886 from taosdata/feature/tfs 

Feature/tfs
This commit is contained in:
Shengliang Guan 2022-01-18 17:32:12 +08:00 committed by GitHub
parent 8237e15e31 d6359f3ab1
commit 7c77464dc1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 314 additions and 484 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
include/libs/tfs/tfs.h
View File

@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TD_TFS_H
#define TD_TFS_H
#ifndef _TD_TFS_H_
#define _TD_TFS_H_
#include "tglobal.h"
@ -23,8 +23,8 @@ extern "C" {
#endif
typedef struct {
int level;
int id;
int32_t level;
int32_t id;
} SDiskID;
#define TFS_UNDECIDED_LEVEL -1
@ -36,33 +36,25 @@ typedef struct {
// FS APIs ====================================
typedef struct {
int64_t tsize;
int64_t total;
int64_t used;
int64_t avail;
} SFSMeta;
typedef struct {
int64_t size;
int64_t used;
int64_t free;
int16_t nAvailDisks; // # of Available disks
} STierMeta;
int tfsInit(SDiskCfg *pDiskCfg, int ndisk);
void tfsCleanup();
void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numLevels);
void tfsGetMeta(SFSMeta *pMeta);
void tfsAllocDisk(int expLevel, int *level, int *id);
int32_t tfsInit(SDiskCfg *pDiskCfg, int32_t ndisk);
void tfsCleanup();
void tfsUpdateSize(SFSMeta *pFSMeta);
void tfsAllocDisk(int32_t expLevel, int32_t *level, int32_t *id);
const char *TFS_PRIMARY_PATH();
const char *TFS_DISK_PATH(int level, int id);
const char *TFS_DISK_PATH(int32_t level, int32_t id);
// TFILE APIs ====================================
typedef struct {
int level;
int id;
char rname[TSDB_FILENAME_LEN]; // REL name
char aname[TSDB_FILENAME_LEN]; // ABS name
int32_t level;
int32_t id;
char rname[TSDB_FILENAME_LEN]; // REL name
char aname[TSDB_FILENAME_LEN]; // ABS name
} TFILE;
#define TFILE_LEVEL(pf) ((pf)->level)
@ -76,23 +68,23 @@ typedef struct {
#define tfscopy(sf, df) taosCopyFile(TFILE_NAME(sf), TFILE_NAME(df))
#define tfsrename(sf, df) taosRename(TFILE_NAME(sf), TFILE_NAME(df))
void tfsInitFile(TFILE *pf, int level, int id, const char *bname);
bool tfsIsSameFile(const TFILE *pf1, const TFILE *pf2);
int tfsEncodeFile(void **buf, TFILE *pf);
void *tfsDecodeFile(void *buf, TFILE *pf);
void tfsbasename(const TFILE *pf, char *dest);
void tfsdirname(const TFILE *pf, char *dest);
void tfsInitFile(TFILE *pf, int32_t level, int32_t id, const char *bname);
bool tfsIsSameFile(const TFILE *pf1, const TFILE *pf2);
int32_t tfsEncodeFile(void **buf, TFILE *pf);
void *tfsDecodeFile(void *buf, TFILE *pf);
void tfsbasename(const TFILE *pf, char *dest);
void tfsdirname(const TFILE *pf, char *dest);
// DIR APIs ====================================
int tfsMkdirAt(const char *rname, int level, int id);
int tfsMkdirRecurAt(const char *rname, int level, int id);
int tfsMkdir(const char *rname);
int tfsRmdir(const char *rname);
int tfsRename(char *orname, char *nrname);
int32_t tfsMkdirAt(const char *rname, int32_t level, int32_t id);
int32_t tfsMkdirRecurAt(const char *rname, int32_t level, int32_t id);
int32_t tfsMkdir(const char *rname);
int32_t tfsRmdir(const char *rname);
int32_t tfsRename(char *orname, char *nrname);
typedef struct TDIR TDIR;
TDIR * tfsOpendir(const char *rname);
TDIR *tfsOpendir(const char *rname);
const TFILE *tfsReaddir(TDIR *tdir);
void tfsClosedir(TDIR *tdir);
@ -100,4 +92,4 @@ void tfsClosedir(TDIR *tdir);
}
#endif
#endif
#endif /*_TD_TFS_H_*/

6
include/os/osSysinfo.h
View File

@ -35,12 +35,12 @@ extern char tsLocale[];
extern char tsCharset[]; // default encode string
typedef struct {
int64_t tsize;
int64_t total;
int64_t used;
int64_t avail;
} SysDiskSize;
} SDiskSize;
int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize);
int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
int32_t taosGetCpuCores();
void taosGetSystemInfo();
bool taosReadProcIO(int64_t *rchars, int64_t *wchars);

2
include/util/taoserror.h
View File

@ -411,7 +411,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004) //"WAL out of memory")
// tfs
#define TSDB_CODE_FS_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x2200) //"tfs out of memory")
#define TSDB_CODE_FS_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x2200) //"tfs out of memory")
#define TSDB_CODE_FS_INVLD_CFG TAOS_DEF_ERROR_CODE(0, 0x2201) //"tfs invalid mount config")
#define TSDB_CODE_FS_TOO_MANY_MOUNT TAOS_DEF_ERROR_CODE(0, 0x2202) //"tfs too many mount")
#define TSDB_CODE_FS_DUP_PRIMARY TAOS_DEF_ERROR_CODE(0, 0x2203) //"tfs duplicate primary mount")

85
source/libs/tfs/inc/tfsint.h → source/libs/tfs/inc/tfsInt.h
View File

@ -13,19 +13,20 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TD_TFSINT_H
#define TD_TFSINT_H
#ifndef _TD_TFS_INT_H_
#define _TD_TFS_INT_H_
#include "tlog.h"
#include "tglobal.h"
#include "tfs.h"
#include "os.h"
#include "taosdef.h"
#include "taoserror.h"
#include "tcoding.h"
#include "tfs.h"
#include "tglobal.h"
#include "thash.h"
#include "tlog.h"
#ifdef __cplusplus
extern "C" {
#endif
extern int fsDebugFlag;
extern int32_t fsDebugFlag;
// For debug purpose
#define fFatal(...) { if (fsDebugFlag & DEBUG_FATAL) { taosPrintLog("TFS FATAL ", 255, __VA_ARGS__); }}
@ -38,60 +39,44 @@ extern int fsDebugFlag;
// Global Definitions
#define TFS_MIN_DISK_FREE_SIZE 50 * 1024 * 1024
// tdisk.c ======================================================
typedef struct {
int64_t size;
int64_t used;
int64_t free;
} SDiskMeta;
typedef struct SDisk {
int level;
int id;
char dir[TSDB_FILENAME_LEN];
SDiskMeta dmeta;
int32_t level;
int32_t id;
char *path;
SDiskSize size;
} SDisk;
#define DISK_LEVEL(pd) ((pd)->level)
#define DISK_ID(pd) ((pd)->id)
#define DISK_DIR(pd) ((pd)->dir)
#define DISK_META(pd) ((pd)->dmeta)
#define DISK_SIZE(pd) ((pd)->dmeta.size)
#define DISK_USED_SIZE(pd) ((pd)->dmeta.used)
#define DISK_FREE_SIZE(pd) ((pd)->dmeta.free)
SDisk *tfsNewDisk(int level, int id, const char *dir);
SDisk *tfsFreeDisk(SDisk *pDisk);
int tfsUpdateDiskInfo(SDisk *pDisk);
// ttier.c ======================================================
typedef struct STier {
pthread_spinlock_t lock;
int level;
int16_t ndisk; // # of disks mounted to this tier
int16_t nextid; // next disk id to allocate
STierMeta tmeta;
SDisk * disks[TSDB_MAX_DISKS_PER_TIER];
int32_t level;
int16_t nextid; // next disk id to allocate
int16_t ndisk; // # of disks mounted to this tier
int16_t nAvailDisks; // # of Available disks
SDisk *disks[TSDB_MAX_DISKS_PER_TIER];
SDiskSize size;
} STier;
#define TIER_LEVEL(pt) ((pt)->level)
#define TIER_NDISKS(pt) ((pt)->ndisk)
#define TIER_SIZE(pt) ((pt)->tmeta.size)
#define TIER_FREE_SIZE(pt) ((pt)->tmeta.free)
#define TIER_AVAIL_DISKS(pt) ((pt)->tmeta.nAvailDisks)
#define DISK_AT_TIER(pt, id) ((pt)->disks[id])
int tfsInitTier(STier *pTier, int level);
void tfsDestroyTier(STier *pTier);
SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg);
void tfsUpdateTierInfo(STier *pTier, STierMeta *pTierMeta);
int tfsAllocDiskOnTier(STier *pTier);
void tfsGetTierMeta(STier *pTier, STierMeta *pTierMeta);
void tfsPosNextId(STier *pTier);
#define DISK_AT_TIER(pt, id) ((pt)->disks[id])
#define DISK_DIR(pd) ((pd)->path)
SDisk *tfsNewDisk(int32_t level, int32_t id, const char *dir);
SDisk *tfsFreeDisk(SDisk *pDisk);
int32_t tfsUpdateDiskSize(SDisk *pDisk);
int32_t tfsInitTier(STier *pTier, int32_t level);
void tfsDestroyTier(STier *pTier);
SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg);
void tfsUpdateTierSize(STier *pTier);
int32_t tfsAllocDiskOnTier(STier *pTier);
void tfsPosNextId(STier *pTier);
#ifdef __cplusplus
}
#endif
#endif
#endif /*_TD_TFS_INT_H_*/

171
source/libs/tfs/src/tfs.c
View File

@ -13,22 +13,17 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "taosdef.h"
#include "taoserror.h"
#include "tfs.h"
#include "tfsint.h"
#include "thash.h"
#define _DEFAULT_SOURCE
#include "tfsInt.h"
#define TMPNAME_LEN (TSDB_FILENAME_LEN * 2 + 32)
typedef struct {
pthread_spinlock_t lock;
SFSMeta meta;
int nlevel;
int32_t nlevel;
STier tiers[TSDB_MAX_TIERS];
SHashObj * map; // name to did map
SHashObj *map; // name to did map
} SFS;
typedef struct {
@ -52,21 +47,24 @@ static SFS tfs = {0};
static SFS *pfs = &tfs;
// STATIC DECLARATION
static int tfsMount(SDiskCfg *pCfg);
static int tfsCheck();
static int tfsCheckAndFormatCfg(SDiskCfg *pCfg);
static int tfsFormatDir(char *idir, char *odir);
static SDisk *tfsGetDiskByID(SDiskID did);
static SDisk *tfsGetDiskByName(const char *dir);
static int tfsOpendirImpl(TDIR *tdir);
static void tfsInitDiskIter(SDiskIter *pIter);
static SDisk *tfsNextDisk(SDiskIter *pIter);
static int32_t tfsMount(SDiskCfg *pCfg);
static int32_t tfsCheck();
static int32_t tfsCheckAndFormatCfg(SDiskCfg *pCfg);
static int32_t tfsFormatDir(char *idir, char *odir);
static SDisk *tfsGetDiskByID(SDiskID did);
static SDisk *tfsGetDiskByName(const char *dir);
static int32_t tfsOpendirImpl(TDIR *tdir);
static void tfsInitDiskIter(SDiskIter *pIter);
static SDisk *tfsNextDisk(SDiskIter *pIter);
// FS APIs ====================================
int tfsInit(SDiskCfg *pDiskCfg, int ndisk) {
ASSERT(ndisk > 0);
int32_t tfsInit(SDiskCfg *pDiskCfg, int32_t ndisk) {
if (ndisk < 0) {
terrno = TSDB_CODE_INVALID_PARA;
return -1;
}
for (int level = 0; level < TSDB_MAX_TIERS; level++) {
for (int32_t level = 0; level < TSDB_MAX_TIERS; level++) {
if (tfsInitTier(TFS_TIER_AT(level), level) < 0) {
while (true) {
level--;
@ -84,12 +82,12 @@ int tfsInit(SDiskCfg *pDiskCfg, int ndisk) {
pfs->map = taosHashInit(TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER * 2,
taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
if (pfs->map == NULL) {
terrno = TSDB_CODE_FS_OUT_OF_MEMORY;
terrno = TSDB_CODE_OUT_OF_MEMORY;
tfsCleanup();
return -1;
}
for (int idisk = 0; idisk < ndisk; idisk++) {
for (int32_t idisk = 0; idisk < ndisk; idisk++) {
if (tfsMount(pDiskCfg + idisk) < 0) {
tfsCleanup();
return -1;
@ -101,8 +99,8 @@ int tfsInit(SDiskCfg *pDiskCfg, int ndisk) {
return -1;
}
tfsUpdateInfo(NULL, NULL, 0);
for (int level = 0; level < TFS_NLEVEL(); level++) {
tfsUpdateSize(NULL);
for (int32_t level = 0; level < TFS_NLEVEL(); level++) {
tfsPosNextId(TFS_TIER_AT(level));
}
@ -114,32 +112,27 @@ void tfsCleanup() {
pfs->map = NULL;
pthread_spin_destroy(&(pfs->lock));
for (int level = 0; level < TFS_NLEVEL(); level++) {
for (int32_t level = 0; level < TFS_NLEVEL(); level++) {
tfsDestroyTier(TFS_TIER_AT(level));
}
}
void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numTiers) {
SFSMeta fsMeta;
STierMeta tierMeta;
void tfsUpdateSize(SFSMeta *pFSMeta) {
SFSMeta fsMeta = {0};
SDiskSize size = {0};
if (pFSMeta == NULL) {
pFSMeta = &fsMeta;
}
memset(pFSMeta, 0, sizeof(*pFSMeta));
for (int level = 0; level < TFS_NLEVEL(); level++) {
STierMeta *pTierMeta = &tierMeta;
if (tierMetas && level < numTiers) {
pTierMeta = tierMetas + level;
}
memset(pFSMeta, 0, sizeof(SFSMeta));
for (int32_t level = 0; level < TFS_NLEVEL(); level++) {
STier *pTier = TFS_TIER_AT(level);
tfsUpdateTierInfo(pTier, pTierMeta);
pFSMeta->tsize += pTierMeta->size;
pFSMeta->avail += pTierMeta->free;
pFSMeta->used += pTierMeta->used;
tfsUpdateTierSize(pTier);
pFSMeta->total += pTier->size.total;
pFSMeta->avail += pTier->size.avail;
pFSMeta->used += pTier->size.used;
}
tfsLock();
@ -147,17 +140,9 @@ void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numTiers) {
tfsUnLock();
}
void tfsGetMeta(SFSMeta *pMeta) {
ASSERT(pMeta);
tfsLock();
*pMeta = pfs->meta;
tfsUnLock();
}
/* Allocate an existing available tier level
*/
void tfsAllocDisk(int expLevel, int *level, int *id) {
void tfsAllocDisk(int32_t expLevel, int32_t *level, int32_t *id) {
ASSERT(expLevel >= 0);
*level = expLevel;
@ -182,10 +167,10 @@ void tfsAllocDisk(int expLevel, int *level, int *id) {
}
const char *TFS_PRIMARY_PATH() { return DISK_DIR(TFS_PRIMARY_DISK()); }
const char *TFS_DISK_PATH(int level, int id) { return DISK_DIR(TFS_DISK_AT(level, id)); }
const char *TFS_DISK_PATH(int32_t level, int32_t id) { return DISK_DIR(TFS_DISK_AT(level, id)); }
// TFILE APIs ====================================
void tfsInitFile(TFILE *pf, int level, int id, const char *bname) {
void tfsInitFile(TFILE *pf, int32_t level, int32_t id, const char *bname) {
ASSERT(TFS_IS_VALID_DISK(level, id));
SDisk *pDisk = TFS_DISK_AT(level, id);
@ -208,8 +193,8 @@ bool tfsIsSameFile(const TFILE *pf1, const TFILE *pf2) {
return true;
}
int tfsEncodeFile(void **buf, TFILE *pf) {
int tlen = 0;
int32_t tfsEncodeFile(void **buf, TFILE *pf) {
int32_t tlen = 0;
tlen += taosEncodeVariantI32(buf, pf->level);
tlen += taosEncodeVariantI32(buf, pf->id);
@ -220,7 +205,7 @@ int tfsEncodeFile(void **buf, TFILE *pf) {
void *tfsDecodeFile(void *buf, TFILE *pf) {
int32_t level, id;
char * rname;
char *rname;
buf = taosDecodeVariantI32(buf, &(level));
buf = taosDecodeVariantI32(buf, &(id));
@ -247,7 +232,7 @@ void tfsdirname(const TFILE *pf, char *dest) {
}
// DIR APIs ====================================
int tfsMkdirAt(const char *rname, int level, int id) {
int32_t tfsMkdirAt(const char *rname, int32_t level, int32_t id) {
SDisk *pDisk = TFS_DISK_AT(level, id);
char aname[TMPNAME_LEN];
@ -260,7 +245,7 @@ int tfsMkdirAt(const char *rname, int level, int id) {
return 0;
}
int tfsMkdirRecurAt(const char *rname, int level, int id) {
int32_t tfsMkdirRecurAt(const char *rname, int32_t level, int32_t id) {
if (tfsMkdirAt(rname, level, id) < 0) {
if (errno == ENOENT) {
// Try to create upper
@ -293,10 +278,10 @@ int tfsMkdirRecurAt(const char *rname, int level, int id) {
return 0;
}
int tfsMkdir(const char *rname) {
for (int level = 0; level < TFS_NLEVEL(); level++) {
int32_t tfsMkdir(const char *rname) {
for (int32_t level = 0; level < TFS_NLEVEL(); level++) {
STier *pTier = TFS_TIER_AT(level);
for (int id = 0; id < TIER_NDISKS(pTier); id++) {
for (int32_t id = 0; id < TIER_NDISKS(pTier); id++) {
if (tfsMkdirAt(rname, level, id) < 0) {
return -1;
}
@ -306,15 +291,15 @@ int tfsMkdir(const char *rname) {
return 0;
}
int tfsRmdir(const char *rname) {
int32_t tfsRmdir(const char *rname) {
char aname[TMPNAME_LEN] = "\0";
for (int level = 0; level < TFS_NLEVEL(); level++) {
for (int32_t level = 0; level < TFS_NLEVEL(); level++) {
STier *pTier = TFS_TIER_AT(level);
for (int id = 0; id < TIER_NDISKS(pTier); id++) {
SDisk *pDisk = DISK_AT_TIER(pTier, id);
for (int32_t id = 0; id < TIER_NDISKS(pTier); id++) {
SDisk *pDisk = pTier->disks[id];
snprintf(aname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), rname);
snprintf(aname, TMPNAME_LEN, "%s%s%s", DISK_DIR(pDisk), TS_PATH_DELIMITER, rname);
taosRemoveDir(aname);
}
@ -323,13 +308,14 @@ int tfsRmdir(const char *rname) {
return 0;
}
int tfsRename(char *orname, char *nrname) {
#if 0
int32_t tfsRename(char *orname, char *nrname) {
char oaname[TMPNAME_LEN] = "\0";
char naname[TMPNAME_LEN] = "\0";
for (int level = 0; level < pfs->nlevel; level++) {
for (int32_t level = 0; level < pfs->nlevel; level++) {
STier *pTier = TFS_TIER_AT(level);
for (int id = 0; id < TIER_NDISKS(pTier); id++) {
for (int32_t id = 0; id < TIER_NDISKS(pTier); id++) {
SDisk *pDisk = DISK_AT_TIER(pTier, id);
snprintf(oaname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), orname);
@ -341,20 +327,21 @@ int tfsRename(char *orname, char *nrname) {
return 0;
}
#endif
struct TDIR {
SDiskIter iter;
int level;
int id;
int32_t level;
int32_t id;
char dirname[TSDB_FILENAME_LEN];
TFILE tfile;
DIR * dir;
DIR *dir;
};
TDIR *tfsOpendir(const char *rname) {
TDIR *tdir = (TDIR *)calloc(1, sizeof(*tdir));
if (tdir == NULL) {
terrno = TSDB_CODE_FS_OUT_OF_MEMORY;
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
@ -407,9 +394,9 @@ void tfsClosedir(TDIR *tdir) {
}
// private
static int tfsMount(SDiskCfg *pCfg) {
static int32_t tfsMount(SDiskCfg *pCfg) {
SDiskID did;
SDisk * pDisk = NULL;
SDisk *pDisk = NULL;
if (tfsCheckAndFormatCfg(pCfg) < 0) return -1;
@ -419,7 +406,7 @@ static int tfsMount(SDiskCfg *pCfg) {
fError("failed to mount disk %s to level %d since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
return -1;
}
did.id = DISK_ID(pDisk);
did.id = pDisk->id;
taosHashPut(pfs->map, (void *)(pCfg->dir), strnlen(pCfg->dir, TSDB_FILENAME_LEN), (void *)(&did), sizeof(did));
if (pfs->nlevel < pCfg->level + 1) pfs->nlevel = pCfg->level + 1;
@ -427,7 +414,7 @@ static int tfsMount(SDiskCfg *pCfg) {
return 0;
}
static int tfsCheckAndFormatCfg(SDiskCfg *pCfg) {
static int32_t tfsCheckAndFormatCfg(SDiskCfg *pCfg) {
char dirName[TSDB_FILENAME_LEN] = "\0";
struct stat pstat;
@ -486,10 +473,10 @@ static int tfsCheckAndFormatCfg(SDiskCfg *pCfg) {
return 0;
}
static int tfsFormatDir(char *idir, char *odir) {
static int32_t tfsFormatDir(char *idir, char *odir) {
wordexp_t wep = {0};
int code = wordexp(idir, &wep, 0);
int32_t code = wordexp(idir, &wep, 0);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(code);
return -1;
@ -507,14 +494,14 @@ static int tfsFormatDir(char *idir, char *odir) {
return 0;
}
static int tfsCheck() {
static int32_t tfsCheck() {
if (TFS_PRIMARY_DISK() == NULL) {
fError("no primary disk is set");
terrno = TSDB_CODE_FS_NO_PRIMARY_DISK;
return -1;
}
for (int level = 0; level < TFS_NLEVEL(); level++) {
for (int32_t level = 0; level < TFS_NLEVEL(); level++) {
if (TIER_NDISKS(TFS_TIER_AT(level)) == 0) {
fError("no disk at level %d", level);
terrno = TSDB_CODE_FS_NO_MOUNT_AT_TIER;
@ -528,8 +515,8 @@ static int tfsCheck() {
static SDisk *tfsGetDiskByID(SDiskID did) { return TFS_DISK_AT(did.level, did.id); }
static SDisk *tfsGetDiskByName(const char *dir) {
SDiskID did;
SDisk * pDisk = NULL;
void * pr = NULL;
SDisk *pDisk = NULL;
void *pr = NULL;
pr = taosHashGet(pfs->map, (void *)dir, strnlen(dir, TSDB_FILENAME_LEN));
if (pr == NULL) return NULL;
@ -541,7 +528,7 @@ static SDisk *tfsGetDiskByName(const char *dir) {
return pDisk;
}
static int tfsOpendirImpl(TDIR *tdir) {
static int32_t tfsOpendirImpl(TDIR *tdir) {
SDisk *pDisk = NULL;
char adir[TMPNAME_LEN * 2] = "\0";
@ -554,10 +541,10 @@ static int tfsOpendirImpl(TDIR *tdir) {
pDisk = tfsNextDisk(&(tdir->iter));
if (pDisk == NULL) return 0;
tdir->level = DISK_LEVEL(pDisk);
tdir->id = DISK_ID(pDisk);
tdir->level = pDisk->level;
tdir->id = pDisk->id;
snprintf(adir, TMPNAME_LEN * 2, "%s/%s", DISK_DIR(pDisk), tdir->dirname);
snprintf(adir, TMPNAME_LEN * 2, "%s%s%s", pDisk->path, TS_PATH_DELIMITER,tdir->dirname);
tdir->dir = opendir(adir);
if (tdir->dir != NULL) break;
}
@ -572,8 +559,8 @@ static SDisk *tfsNextDisk(SDiskIter *pIter) {
if (pDisk == NULL) return NULL;
int level = DISK_LEVEL(pDisk);
int id = DISK_ID(pDisk);
int32_t level = pDisk->level;
int32_t id = pDisk->id;
id++;
if (id < TIER_NDISKS(TFS_TIER_AT(level))) {
@ -596,21 +583,21 @@ static SDisk *tfsNextDisk(SDiskIter *pIter) {
// OTHER FUNCTIONS ===================================
void taosGetDisk() {
const double unit = 1024 * 1024 * 1024;
SysDiskSize diskSize;
SDiskSize diskSize;
SFSMeta fsMeta;
tfsUpdateInfo(&fsMeta, NULL, 0);
tsTotalDataDirGB = (float)(fsMeta.tsize / unit);
tfsUpdateSize(&fsMeta);
tsTotalDataDirGB = (float)(fsMeta.total / unit);
tsUsedDataDirGB = (float)(fsMeta.used / unit);
tsAvailDataDirGB = (float)(fsMeta.avail / unit);
if (taosGetDiskSize(tsLogDir, &diskSize) == 0) {
tsTotalLogDirGB = (float)(diskSize.tsize / unit);
tsTotalLogDirGB = (float)(diskSize.total / unit);
tsAvailLogDirGB = (float)(diskSize.avail / unit);
}
if (taosGetDiskSize(tsTempDir, &diskSize) == 0) {
tsTotalTmpDirGB = (float)(diskSize.tsize / unit);
tsTotalTmpDirGB = (float)(diskSize.total / unit);
tsAvailTmpDirectorySpace = (float)(diskSize.avail / unit);
}
}

45
source/libs/tfs/src/tdisk.c → source/libs/tfs/src/tfsDisk.c
View File

@ -12,48 +12,45 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "taoserror.h"
#include "tfsint.h"
#define _DEFAULT_SOURCE
#include "tfsInt.h"
// PROTECTED ====================================
SDisk *tfsNewDisk(int level, int id, const char *dir) {
SDisk *pDisk = (SDisk *)calloc(1, sizeof(*pDisk));
SDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) {
SDisk *pDisk = calloc(1, sizeof(SDisk));
if (pDisk == NULL) {
terrno = TSDB_CODE_FS_OUT_OF_MEMORY;
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pDisk->path = strdup(path);
if (pDisk->path == NULL) {
free(pDisk);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pDisk->level = level;
pDisk->id = id;
tstrncpy(pDisk->dir, dir, TSDB_FILENAME_LEN);
taosGetDiskSize(pDisk->path, &pDisk->size);
return pDisk;
}
SDisk *tfsFreeDisk(SDisk *pDisk) {
if (pDisk) {
if (pDisk != NULL) {
free(pDisk->path);
free(pDisk);
}
return NULL;
}
int tfsUpdateDiskInfo(SDisk *pDisk) {
ASSERT(pDisk != NULL);
SysDiskSize diskSize = {0};
int code = taosGetDiskSize(pDisk->dir, &diskSize);
if (code != 0) {
fError("failed to update disk information at level %d id %d dir %s since %s", pDisk->level, pDisk->id, pDisk->dir,
strerror(errno));
int32_t tfsUpdateDiskSize(SDisk *pDisk) {
if (taosGetDiskSize(pDisk->path, &pDisk->size) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
fError("failed to get disk:%s size, level:%d id:%d since %s", pDisk->path, pDisk->level, pDisk->id, terrstr());
return -1;
}
pDisk->dmeta.size = diskSize.tsize;
pDisk->dmeta.used = diskSize.used;
pDisk->dmeta.free = diskSize.avail;
return code;
return 0;
}

143
source/libs/tfs/src/tfsTier.c Normal file
View File

@ -0,0 +1,143 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "tfsInt.h"
#define tfsLockTier(pTier) pthread_spin_lock(&(pTier)->lock)
#define tfsUnLockTier(pTier) pthread_spin_unlock(&(pTier)->lock)
int32_t tfsInitTier(STier *pTier, int32_t level) {
memset(pTier, 0, sizeof(STier));
if (pthread_spin_init(&pTier->lock, 0) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
pTier->level = level;
return 0;
}
void tfsDestroyTier(STier *pTier) {
for (int32_t id = 0; id < TSDB_MAX_DISKS_PER_TIER; id++) {
pTier->disks[id] = tfsFreeDisk(pTier->disks[id]);
}
pTier->ndisk = 0;
pthread_spin_destroy(&(pTier->lock));
}
SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) {
if (pTier->ndisk >= TSDB_MAX_DISKS_PER_TIER) {
terrno = TSDB_CODE_FS_TOO_MANY_MOUNT;
return NULL;
}
int32_t id = 0;
if (pTier->level == 0) {
if (pTier->disks[0] != NULL) {
id = pTier->ndisk;
} else {
if (pCfg->primary) {
id = 0;
} else {
id = pTier->ndisk + 1;
}
}
} else {
id = pTier->ndisk;
}
if (id >= TSDB_MAX_DISKS_PER_TIER) {
terrno = TSDB_CODE_FS_TOO_MANY_MOUNT;
return NULL;
}
SDisk *pDisk = tfsNewDisk(pCfg->level, id, pCfg->dir);
if (pDisk == NULL) return NULL;
pTier->disks[id] = pDisk;
pTier->ndisk++;
fInfo("disk %s is mounted to tier level %d id %d", pCfg->dir, pCfg->level, id);
return pTier->disks[id];
}
void tfsUpdateTierSize(STier *pTier) {
SDiskSize size = {0};
int16_t nAvailDisks = 0;
tfsLockTier(pTier);
for (int32_t id = 0; id < pTier->ndisk; id++) {
SDisk *pDisk = pTier->disks[id];
if (pDisk == NULL) continue;
size.total += pDisk->size.total;
size.used += pDisk->size.used;
size.avail += pDisk->size.avail;
nAvailDisks++;
}
pTier->size = size;
pTier->nAvailDisks = nAvailDisks;
tfsUnLockTier(pTier);
}
// Round-Robin to allocate disk on a tier
int32_t tfsAllocDiskOnTier(STier *pTier) {
terrno = TSDB_CODE_FS_NO_VALID_DISK;
tfsLockTier(pTier);
if (pTier->ndisk <= 0 || pTier->nAvailDisks <= 0) {
tfsUnLockTier(pTier);
return -1;
}
int32_t retId = -1;
for (int32_t id = 0; id < TSDB_MAX_DISKS_PER_TIER; ++id) {
int32_t diskId = (pTier->nextid + id) % pTier->ndisk;
SDisk *pDisk = pTier->disks[diskId];
if (pDisk == NULL) continue;
if (pDisk->size.avail < TFS_MIN_DISK_FREE_SIZE) continue;
retId = diskId;
terrno = 0;
pTier->nextid = (diskId + 1) % pTier->ndisk;
break;
}
tfsUnLockTier(pTier);
return retId;
}
void tfsPosNextId(STier *pTier) {
int32_t nextid = 0;
for (int32_t id = 1; id < pTier->ndisk; id++) {
SDisk *pLDisk = pTier->disks[nextid];
SDisk *pDisk = pTier->disks[id];
if (pDisk->size.avail > TFS_MIN_DISK_FREE_SIZE && pDisk->size.avail > pLDisk->size.avail) {
nextid = id;
}
}
pTier->nextid = nextid;
}

170
source/libs/tfs/src/ttier.c
View File

@ -1,170 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "taosdef.h"
#include "taoserror.h"
#include "tfsint.h"
#define tfsLockTier(pTier) pthread_spin_lock(&((pTier)->lock))
#define tfsUnLockTier(pTier) pthread_spin_unlock(&((pTier)->lock))
// PROTECTED ==========================================
int tfsInitTier(STier *pTier, int level) {
memset((void *)pTier, 0, sizeof(*pTier));
int code = pthread_spin_init(&(pTier->lock), 0);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
return -1;
}
pTier->level = level;
return 0;
}
void tfsDestroyTier(STier *pTier) {
for (int id = 0; id < TSDB_MAX_DISKS_PER_TIER; id++) {
DISK_AT_TIER(pTier, id) = tfsFreeDisk(DISK_AT_TIER(pTier, id));
}
pTier->ndisk = 0;
pthread_spin_destroy(&(pTier->lock));
}
SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) {
ASSERT(pTier->level == pCfg->level);
int id = 0;
SDisk *pDisk;
if (TIER_NDISKS(pTier) >= TSDB_MAX_DISKS_PER_TIER) {
terrno = TSDB_CODE_FS_TOO_MANY_MOUNT;
return NULL;
}
if (pTier->level == 0) {
if (DISK_AT_TIER(pTier, 0) != NULL) {
id = pTier->ndisk;
} else {
if (pCfg->primary) {
id = 0;
} else {
id = pTier->ndisk + 1;
}
if (id >= TSDB_MAX_DISKS_PER_TIER) {
terrno = TSDB_CODE_FS_TOO_MANY_MOUNT;
return NULL;
}
}
} else {
id = pTier->ndisk;
}
pDisk = tfsNewDisk(pCfg->level, id, pCfg->dir);
if (pDisk == NULL) return NULL;
DISK_AT_TIER(pTier, id) = pDisk;
pTier->ndisk++;
fInfo("disk %s is mounted to tier level %d id %d", pCfg->dir, pCfg->level, id);
return DISK_AT_TIER(pTier, id);
}
void tfsUpdateTierInfo(STier *pTier, STierMeta *pTierMeta) {
STierMeta tmeta;
if (pTierMeta == NULL) {
pTierMeta = &tmeta;
}
memset(pTierMeta, 0, sizeof(*pTierMeta));
tfsLockTier(pTier);
for (int id = 0; id < pTier->ndisk; id++) {
if (tfsUpdateDiskInfo(DISK_AT_TIER(pTier, id)) < 0) {
continue;
}
pTierMeta->size += DISK_SIZE(DISK_AT_TIER(pTier, id));
pTierMeta->used += DISK_USED_SIZE(DISK_AT_TIER(pTier, id));
pTierMeta->free += DISK_FREE_SIZE(DISK_AT_TIER(pTier, id));
pTierMeta->nAvailDisks++;
}
pTier->tmeta = *pTierMeta;
tfsUnLockTier(pTier);
}
// Round-Robin to allocate disk on a tier
int tfsAllocDiskOnTier(STier *pTier) {
ASSERT(pTier->ndisk > 0);
int id = TFS_UNDECIDED_ID;
SDisk *pDisk;
tfsLockTier(pTier);
if (TIER_AVAIL_DISKS(pTier) <= 0) {
tfsUnLockTier(pTier);
return id;
}
id = pTier->nextid;
while (true) {
pDisk = DISK_AT_TIER(pTier, id);
ASSERT(pDisk != NULL);
if (DISK_FREE_SIZE(pDisk) < TFS_MIN_DISK_FREE_SIZE) {
id = (id + 1) % pTier->ndisk;
if (id == pTier->nextid) {
tfsUnLockTier(pTier);
return TFS_UNDECIDED_ID;
} else {
continue;
}
} else {
pTier->nextid = (id + 1) % pTier->ndisk;
break;
}
}
tfsUnLockTier(pTier);
return id;
}
void tfsGetTierMeta(STier *pTier, STierMeta *pTierMeta) {
ASSERT(pTierMeta != NULL);
tfsLockTier(pTier);
*pTierMeta = pTier->tmeta;
tfsUnLockTier(pTier);
}
void tfsPosNextId(STier *pTier) {
ASSERT(pTier->ndisk > 0);
int nextid = 0;
for (int id = 1; id < pTier->ndisk; id++) {
SDisk *pLDisk = DISK_AT_TIER(pTier, nextid);
SDisk *pDisk = DISK_AT_TIER(pTier, id);
if (DISK_FREE_SIZE(pDisk) > TFS_MIN_DISK_FREE_SIZE && DISK_FREE_SIZE(pDisk) > DISK_FREE_SIZE(pLDisk)) {
nextid = id;
}
}
pTier->nextid = nextid;
}

9
source/os/src/osSysinfo.c
View File

@ -121,7 +121,7 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) {
return true;
}
int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) {
int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) {
unsigned _int64 i64FreeBytesToCaller;
unsigned _int64 i64TotalBytes;
unsigned _int64 i64FreeBytes;
@ -438,7 +438,7 @@ int taosSystem(const char *cmd) {
void taosSetCoreDump() {}
int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) {
int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) {
struct statvfs info;
if (statvfs(dataDir, &info)) {
//printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno));
@ -771,13 +771,12 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) {
return true;
}
int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) {
int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) {
struct statvfs info;
if (statvfs(dataDir, &info)) {
//printf("failed to get disk size, dataDir:%s errno:%s", dataDir, strerror(errno));
return -1;
} else {
diskSize->tsize = info.f_blocks * info.f_frsize;
diskSize->total = info.f_blocks * info.f_frsize;
diskSize->avail = info.f_bavail * info.f_frsize;
diskSize->used = (info.f_blocks - info.f_bfree) * info.f_frsize;
return 0;

2
source/util/src/terror.c
View File

@ -396,7 +396,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, "WAL file is corrupted
TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, "WAL size exceeds limit")
// tfs
TAOS_DEFINE_ERROR(TSDB_CODE_FS_OUT_OF_MEMORY, "tfs out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_FS_APP_ERROR, "tfs out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_CFG, "tfs invalid mount config")
TAOS_DEFINE_ERROR(TSDB_CODE_FS_TOO_MANY_MOUNT, "tfs too many mount")
TAOS_DEFINE_ERROR(TSDB_CODE_FS_DUP_PRIMARY, "tfs duplicate primary mount")

103
src/inc/tfs.h
View File

@ -1,103 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TD_TFS_H
#define TD_TFS_H
#include "tglobal.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct {
int level;
int id;
} SDiskID;
#define TFS_UNDECIDED_LEVEL -1
#define TFS_UNDECIDED_ID -1
#define TFS_PRIMARY_LEVEL 0
#define TFS_PRIMARY_ID 0
#define TFS_MIN_LEVEL 0
#define TFS_MAX_LEVEL (TSDB_MAX_TIERS - 1)
// FS APIs ====================================
typedef struct {
int64_t tsize;
int64_t used;
int64_t avail;
} SFSMeta;
typedef struct {
int64_t size;
int64_t used;
int64_t free;
int16_t nAvailDisks; // # of Available disks
} STierMeta;
int tfsInit(SDiskCfg *pDiskCfg, int ndisk);
void tfsCleanup();
void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numLevels);
void tfsGetMeta(SFSMeta *pMeta);
void tfsAllocDisk(int expLevel, int *level, int *id);
const char *TFS_PRIMARY_PATH();
const char *TFS_DISK_PATH(int level, int id);
// TFILE APIs ====================================
typedef struct {
int level;
int id;
char rname[TSDB_FILENAME_LEN]; // REL name
char aname[TSDB_FILENAME_LEN]; // ABS name
} TFILE;
#define TFILE_LEVEL(pf) ((pf)->level)
#define TFILE_ID(pf) ((pf)->id)
#define TFILE_NAME(pf) ((pf)->aname)
#define TFILE_REL_NAME(pf) ((pf)->rname)
#define tfsopen(pf, flags) open(TFILE_NAME(pf), flags)
#define tfsclose(fd) close(fd)
#define tfsremove(pf) remove(TFILE_NAME(pf))
#define tfscopy(sf, df) taosCopy(TFILE_NAME(sf), TFILE_NAME(df))
#define tfsrename(sf, df) taosRename(TFILE_NAME(sf), TFILE_NAME(df))
void tfsInitFile(TFILE *pf, int level, int id, const char *bname);
bool tfsIsSameFile(const TFILE *pf1, const TFILE *pf2);
int tfsEncodeFile(void **buf, TFILE *pf);
void *tfsDecodeFile(void *buf, TFILE *pf);
void tfsbasename(const TFILE *pf, char *dest);
void tfsdirname(const TFILE *pf, char *dest);
// DIR APIs ====================================
int tfsMkdirAt(const char *rname, int level, int id);
int tfsMkdirRecurAt(const char *rname, int level, int id);
int tfsMkdir(const char *rname);
int tfsRmdir(const char *rname);
int tfsRename(char *orname, char *nrname);
typedef struct TDIR TDIR;
TDIR * tfsOpendir(const char *rname);
const TFILE *tfsReaddir(TDIR *tdir);
void tfsClosedir(TDIR *tdir);
#ifdef __cplusplus
}
#endif
#endif