xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor

This commit is contained in:
Hongze Cheng 2020-11-18 18:15:52 +08:00
parent dfebc184ab
commit 4f7c4ef622
3 changed files with 170 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
src/common/inc/tdisk.h
View File

@ -26,81 +26,10 @@
extern "C" {
#endif
typedef struct {
int level;
int did;
} SDiskID;
typedef struct {
uint64_t size;
uint64_t free;
uint64_t nfiles;
} SDiskMeta;
typedef struct {
uint64_t tsize;
uint64_t avail; // bytes
} STiersMeta;
typedef struct {
int level;
int did;
char dir[TSDB_FILENAME_LEN];
SDiskMeta dmeta;
} SDisk;
typedef struct {
int level;
int nDisks;
SDisk *disks[TSDB_MAX_DISKS_PER_TIER];
} STier;
typedef struct SDnodeTier {
pthread_mutex_t lock;
STiersMeta meta;
int nTiers;
STier tiers[TSDB_MAX_TIERS];
SHashObj * map;
} SDnodeTier;
extern struct SDnodeTier *tsDnodeTier;
#define DNODE_PRIMARY_DISK(pDnodeTier) (pDnodeTier)->tiers[0].disks[0]
static FORCE_INLINE int tdLockTiers(SDnodeTier *pDnodeTier) {
int code = pthread_mutex_lock(&(pDnodeTier->lock));
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(code);
return -1;
}
return 0;
}
static FORCE_INLINE int tdUnLockTiers(SDnodeTier *pDnodeTier) {
int code = pthread_mutex_unlock(&(pDnodeTier->lock));
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(code);
return -1;
}
return 0;
}
static FORCE_INLINE SDisk *tdGetDisk(SDnodeTier *pDnodeTier, int level, int did) {
if (level < 0 || level >= pDnodeTier->nTiers) return NULL;
if (did < 0 || did >= pDnodeTier->tiers[level].nDisks) return NULL;
return pDnodeTier->tiers[level].disks[did];
}
SDnodeTier *tdNewTier();
void * tdCloseTier(SDnodeTier *pDnodeTier);
int tdAddDisks(SDnodeTier *pDnodeTier, SDiskCfg *pDiskCfgs, int ndisks);
int tdUpdateTiersInfo(SDnodeTier *pDnodeTier);
int tdCheckTiers(SDnodeTier *pDnodeTier);
SDisk * tdAssignDisk(SDnodeTier *pDnodeTier, int level);
SDisk * tdGetDiskByName(SDnodeTier *pDnodeTier, char *dirName);
void tdIncDiskFiles(SDnodeTier *pDnodeTier, SDisk *pDisk, bool lock);
void tdDecDiskFiles(SDnodeTier *pDnodeTier, SDisk *pDisk, bool lock);
int tdInitTiers(SDiskCfg *pDiskCfg, int ndisk);
void tdDestroyTiers();
int tdUpdateDiskInfos();
void tdGetPrimaryPath(char *dst);
#ifdef __cplusplus
}

272
src/common/src/tdisk.c
View File

@ -17,111 +17,139 @@
#include "tdisk.h"
#include "tulog.h"
typedef struct {
int level;
int did;
} SDiskID;
typedef struct {
uint64_t size;
uint64_t free;
uint64_t nfiles;
} SDiskMeta;
typedef struct {
uint64_t tsize;
uint64_t avail; // bytes
} STiersMeta;
typedef struct {
int level;
int did;
char dir[TSDB_FILENAME_LEN];
SDiskMeta dmeta;
} SDisk;
typedef struct {
int level;
int nDisks;
SDisk *disks[TSDB_MAX_DISKS_PER_TIER];
} STier;
typedef struct STiers {
pthread_mutex_t lock;
STiersMeta meta;
int nLevel;
STier tiers[TSDB_MAX_TIERS];
SHashObj * map;
} STiers;
#define DISK_MIN_FREE_SPACE 30 * 1024 * 1024 // disk free space less than 100M will not create new file again
#define DNODE_DISK_AVAIL(pDisk) ((pDisk)->dmeta.free > DISK_MIN_FREE_SPACE)
#define TIER_AT_LEVEL(level) (pTiers->tiers + (level))
#define DISK_AT(level, did) (TIER_AT_LEVEL(level)->disks[(did)])
static int tdFormatDir(char *idir, char *odir);
static int tdCheckDisk(char *dirName, int level, int primary);
static int tdUpdateDiskMeta(SDisk *pDisk);
static int tdAddDisk(SDnodeTier *pDnodeTier, char *dir, int level, int primary);
static struct STiers tdTiers;
static struct STiers *pTiers = &tdTiers;
struct SDnodeTier *tsDnodeTier = NULL;
int tdInitTiers(SDiskCfg *pDiskCfg, int ndisk) {
ASSERT(ndisk > 0);
SDnodeTier *tdNewTier() {
SDnodeTier *pDnodeTier = (SDnodeTier *)calloc(1, sizeof(*pDnodeTier));
if (pDnodeTier == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
}
memset((void *)pTiers, 0, sizeof(*pTiers));
int ret = pthread_mutex_init(&(pDnodeTier->lock), NULL);
int ret = pthread_mutex_init(&(pTiers->lock), NULL);
if (ret != 0) {
terrno = TAOS_SYSTEM_ERROR(ret);
tdCloseTier(pDnodeTier);
return NULL;
return -1;
}
pDnodeTier->map = taosHashInit(TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER * 2,
taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
if (pDnodeTier->map == NULL) {
pTiers->map = taosHashInit(TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER * 2,
taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
if (pTiers->map == NULL) {
terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
tdCloseTier(pDnodeTier);
return NULL;
tdDestroyTiers();
return -1;
}
return pDnodeTier;
}
void *tdCloseTier(SDnodeTier *pDnodeTier) {
if (pDnodeTier) {
if (pDnodeTier->map) {
taosHashCleanup(pDnodeTier->map);
pDnodeTier->map = NULL;
for (int idisk = 0; idisk < ndisk; idisk++) {
if (tdAddDisk(pDiskCfg + idisk) < 0) {
tdDestroyTiers();
return -1;
}
pthread_mutex_destroy(&(pDnodeTier->lock));
for (int i = 0; i < pDnodeTier->nTiers; i++) {
STier *pTier = pDnodeTier->tiers + i;
for (int j = 0; j < pTier->nDisks; j++) {
if (pTier->disks[j]) {
free(pTier->disks[j]);
pTier->disks[j] = NULL;
}
}
}
free(pDnodeTier);
}
return NULL;
}
int tdAddDisks(SDnodeTier *pDnodeTier, SDiskCfg *pDiskCfgs, int ndisks) {
ASSERT(ndisks > 0);
for (int i = 0; i < ndisks; i++) {
SDiskCfg *pCfg = pDiskCfgs + i;
tdAddDisk(pDnodeTier, pCfg->dir, pCfg->level, pCfg->primary);
}
if (tdCheckTiers(pDnodeTier) < 0) return -1;
if (tdCheckTiers() < 0) {
tdDestroyTiers();
return -1;
}
return 0;
}
int tdUpdateTiersInfo(SDnodeTier *pDnodeTier) {
tdLockTiers(pDnodeTier);
void tdDestroyTiers() {
taosHashCleanup(pTiers->map);
pTiers->map = NULL;
pDnodeTier->meta.tsize = 0;
pDnodeTier->meta.avail = 0;
pthread_mutex_destroy(&(pTiers->lock));
for (int i = 0; i < pDnodeTier->nTiers; i++) {
STier *pTier = pDnodeTier->tiers + i;
for (int level = TSDB_MAX_TIERS - 1; level >= 0; --level) {
for (int did = TSDB_MAX_DISKS_PER_TIER - 1; did >= 0; --did) {
if (DISK_AT(level, did)) {
free(DISK_AT(level, did));
DISK_AT(level, did) = NULL;
}
}
}
}
int tdUpdateDiskInfos() {
tdLockTiers(pTiers);
pTiers->meta.tsize = 0;
pTiers->meta.avail = 0;
for (int i = 0; i < pTiers->nLevel; i++) {
STier *pTier = pTiers->tiers + i;
for (int j = 0; j < pTier->nDisks; j++) {
SDisk *pDisk = pTier->disks[j];
if (tdUpdateDiskMeta(pDisk) < 0) {
tdUnLockTiers(pDnodeTier);
tdUnLockTiers(pTiers);
return -1;
}
pDnodeTier->meta.tsize += pDisk->dmeta.size;
pDnodeTier->meta.avail += pDisk->dmeta.free;
pTiers->meta.tsize += pDisk->dmeta.size;
pTiers->meta.avail += pDisk->dmeta.free;
}
}
tdUnLockTiers(pDnodeTier);
tdUnLockTiers(pTiers);
return 0;
}
int tdCheckTiers(SDnodeTier *pDnodeTier) {
ASSERT(pDnodeTier->nTiers > 0);
if (DNODE_PRIMARY_DISK(pDnodeTier) == NULL) {
void tdGetPrimaryPath(char *dst) { strncpy(dst, DISK_AT(0, 0)->dir, TSDB_FILENAME_LEN); }
static SDisk *tdGetPrimaryDisk() { return DISK_AT(0, 0); }
static int tdCheckTiers() {
ASSERT(pTiers->nLevel > 0);
if (tdGetPrimaryDisk(pTiers) == NULL) {
terrno = TSDB_CODE_DND_LACK_PRIMARY_DISK;
return -1;
}
for (int i = 0; i < pDnodeTier->nTiers; i++) {
if (pDnodeTier->tiers[i].nDisks == 0) {
for (int i = 0; i < pTiers->nLevel; i++) {
if (pTiers->tiers[i].nDisks == 0) {
terrno = TSDB_CODE_DND_NO_DISK_AT_TIER;
return -1;
}
@ -130,15 +158,15 @@ int tdCheckTiers(SDnodeTier *pDnodeTier) {
return 0;
}
SDisk *tdAssignDisk(SDnodeTier *pDnodeTier, int level) {
ASSERT(level < pDnodeTier->nTiers);
static SDisk *tdAssignDisk(int level) {
ASSERT(level < pTiers->nLevel);
STier *pTier = pDnodeTier->tiers + level;
STier *pTier = pTiers->tiers + level;
SDisk *pDisk = NULL;
ASSERT(pTier->nDisks > 0);
tdLockTiers(pDnodeTier);
tdLockTiers(pTiers);
for (int i = 0; i < pTier->nDisks; i++) {
SDisk *iDisk = pTier->disks[i];
@ -152,18 +180,18 @@ SDisk *tdAssignDisk(SDnodeTier *pDnodeTier, int level) {
if (pDisk == NULL) {
terrno = TSDB_CODE_DND_NO_DISK_SPACE;
tdUnLockTiers(pDnodeTier);
tdUnLockTiers(pTiers);
return NULL;
}
tdIncDiskFiles(pDnodeTier, pDisk, false);
tdIncDiskFiles(pTiers, pDisk, false);
tdUnLockTiers(pDnodeTier);
tdUnLockTiers(pTiers);
return pDisk;
}
SDisk *tdGetDiskByName(SDnodeTier *pDnodeTier, char *dirName) {
static SDisk *tdGetDiskByName(char *dirName) {
char fdirName[TSDB_FILENAME_LEN] = "\0";
SDiskID *pDiskID = NULL;
@ -171,34 +199,34 @@ SDisk *tdGetDiskByName(SDnodeTier *pDnodeTier, char *dirName) {
return NULL;
}
void *ptr = taosHashGet(pDnodeTier->map, (void *)fdirName, strnlen(fdirName, TSDB_FILENAME_LEN));
void *ptr = taosHashGet(pTiers->map, (void *)fdirName, strnlen(fdirName, TSDB_FILENAME_LEN));
if (ptr == NULL) return NULL;
pDiskID = (SDiskID *)ptr;
return tdGetDisk(pDnodeTier, pDiskID->level, pDiskID->did);
return tdGetDisk(pTiers, pDiskID->level, pDiskID->did);
}
void tdIncDiskFiles(SDnodeTier *pDnodeTier, SDisk *pDisk, bool lock) {
static void tdIncDiskFiles(SDisk *pDisk, bool lock) {
if (lock) {
tdLockTiers(pDnodeTier);
tdLockTiers(pTiers);
}
pDisk->dmeta.nfiles++;
if (lock) {
tdUnLockTiers(pDnodeTier);
tdUnLockTiers(pTiers);
}
}
void tdDecDiskFiles(SDnodeTier *pDnodeTier, SDisk *pDisk, bool lock) {
static void tdDecDiskFiles(SDisk *pDisk, bool lock) {
if (lock) {
tdLockTiers(pDnodeTier);
tdLockTiers(pTiers);
}
pDisk->dmeta.nfiles--;
if (lock) {
tdUnLockTiers(pDnodeTier);
tdUnLockTiers(pTiers);
}
}
@ -223,7 +251,7 @@ static int tdFormatDir(char *idir, char *odir) {
return 0;
}
static int tdCheckDisk(char *dirName, int level, int primary) {
static int tdCheckDisk(char *dirName) {
if (access(dirName, W_OK | R_OK | F_OK) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -257,66 +285,66 @@ static int tdUpdateDiskMeta(SDisk *pDisk) {
return 0;
}
static int tdAddDisk(SDnodeTier *pDnodeTier, char *dir, int level, int primary) {
static int tdAddDisk(SDiskCfg *pCfg) {
char dirName[TSDB_FILENAME_LEN] = "\0";
STier * pTier = NULL;
SDiskID diskid = {0};
SDisk * pDisk = NULL;
if (level < 0 || level >= TSDB_MAX_TIERS) {
if (pCfg->level < 0 || pCfg->level >= TSDB_MAX_TIERS) {
terrno = TSDB_CODE_DND_INVALID_DISK_TIER;
uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
return -1;
}
if (tdFormatDir(dir, dirName) < 0) {
uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
if (tdFormatDir(pCfg->dir, dirName) < 0) {
uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
return -1;
}
pTier = pDnodeTier->tiers + level;
diskid.level = level;
pTier = TIER_AT_LEVEL(pCfg->level);
diskid.level = pCfg->level;
if (pTier->nDisks >= TSDB_MAX_DISKS_PER_TIER) {
terrno = TSDB_CODE_DND_TOO_MANY_DISKS;
uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
return -1;
}
if (tdGetDiskByName(pDnodeTier, dirName) != NULL) {
if (tdGetDiskByName(dirName) != NULL) {
terrno = TSDB_CODE_DND_DISK_ALREADY_EXISTS;
uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
return -1;
}
if (tdCheckDisk(dirName, level, primary) < 0) {
uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
if (tdCheckDisk(dirName) < 0) {
uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
return -1;
}
if (primary) {
if (level != 0) {
if (pCfg->primary) {
if (pCfg->level != 0) {
terrno = TSDB_CODE_DND_INVALID_DISK_TIER;
uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
return -1;
}
if (DNODE_PRIMARY_DISK(pDnodeTier) != NULL) {
if (tdGetPrimaryDisk() != NULL) {
terrno = TSDB_CODE_DND_DUPLICATE_PRIMARY_DISK;
uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
return -1;
}
diskid.did = 0;
} else {
if (level == 0) {
if (DNODE_PRIMARY_DISK(pDnodeTier) != NULL) {
if (tdGetPrimaryDisk() != NULL) {
diskid.did = pTier->nDisks;
} else {
diskid.did = pTier->nDisks + 1;
if (diskid.did >= TSDB_MAX_DISKS_PER_TIER) {
terrno = TSDB_CODE_DND_TOO_MANY_DISKS;
uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
return -1;
}
}
@ -328,7 +356,7 @@ static int tdAddDisk(SDnodeTier *pDnodeTier, char *dir, int level, int primary)
pDisk = (SDisk *)calloc(1, sizeof(SDisk));
if (pDisk == NULL) {
terrno = TSDB_CODE_DND_OUT_OF_MEMORY;
uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
return -1;
}
@ -336,27 +364,27 @@ static int tdAddDisk(SDnodeTier *pDnodeTier, char *dir, int level, int primary)
pDisk->level = diskid.level;
pDisk->did = diskid.did;
if (taosHashPut(pDnodeTier->map, (void *)dirName, strnlen(dirName, TSDB_FILENAME_LEN), (void *)(&diskid),
if (taosHashPut(pTiers->map, (void *)dirName, strnlen(dirName, TSDB_FILENAME_LEN), (void *)(&diskid),
sizeof(diskid)) < 0) {
free(pDisk);
terrno = TSDB_CODE_DND_OUT_OF_MEMORY;
uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
return -1;
}
pTier->nDisks++;
pTier->disks[diskid.did] = pDisk;
pDnodeTier->nTiers = MAX(pDnodeTier->nTiers, level + 1);
pTiers->nLevel = MAX(pTiers->nLevel, level + 1);
return 0;
}
void taosGetDisk() {
static void taosGetDisk() {
const double unit = 1024 * 1024 * 1024;
SysDiskSize diskSize;
if (tscEmbedded) {
tdUpdateTiersInfo(tsDnodeTier);
tdUpdateDiskInfos(tsDnodeTier);
tsTotalDataDirGB = (float)tsDnodeTier->meta.tsize / unit;
tsAvailDataDirGB = (float)tsDnodeTier->meta.avail / unit;
}
@ -371,3 +399,29 @@ void taosGetDisk() {
tsAvailTmpDirectorySpace = (float)diskSize.avail / unit;
}
}
static int tdLockTiers() {
int code = pthread_mutex_lock(&(pTiers->lock));
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(code);
return -1;
}
return 0;
}
static int tdUnLockTiers() {
int code = pthread_mutex_unlock(&(pTiers->lock));
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(code);
return -1;
}
return 0;
}
static SDisk *tdGetDisk(int level, int did) {
if (level < 0 || level >= pTiers->nLevel) return NULL;
if (did < 0 || did >= pTiers->tiers[level].nDisks) return NULL;
return pTiers->tiers[level].disks[did];
}

17
src/dnode/src/dnodeMain.c
View File

@ -183,17 +183,11 @@ static void dnodeCheckDataDirOpenned(char *dir) {
}
static int32_t dnodeInitStorage() {
tsDnodeTier = tdNewTier();
if (tsDnodeTier == NULL) {
dError("failed to create new dnode tier since %s", tstrerror(terrno));
return -1;
}
if (tdAddDisks(tsDnodeTier, tsDiskCfg, tsDiskCfgNum) < 0) {
if (tdInitTiers(tsDiskCfg, tsDiskCfgNum) < 0) {
dError("failed to add disks to dnode tier since %s", tstrerror(terrno));
return -1;
}
strncpy(tsDataDir, DNODE_PRIMARY_DISK(tsDnodeTier)->dir, TSDB_FILENAME_LEN);
tdGetPrimaryPath(tsDataDir);
tdGetVnodeRootDir(tsDataDir, tsVnodeDir);
//TODO(dengyihao): no need to init here
@ -236,12 +230,7 @@ static int32_t dnodeInitStorage() {
return 0;
}
static void dnodeCleanupStorage() {
if (tsDnodeTier) {
tdCloseTier(tsDnodeTier);
tsDnodeTier = NULL;
}
}
static void dnodeCleanupStorage() { tdDestroyTiers(); }
bool dnodeIsFirstDeploy() {
return strcmp(tsFirst, tsLocalEp) == 0;