diff --git a/src/common/inc/tdisk.h b/src/common/inc/tdisk.h index 9cc53fa0d2..cf68ccd6e1 100644 --- a/src/common/inc/tdisk.h +++ b/src/common/inc/tdisk.h @@ -26,81 +26,10 @@ extern "C" { #endif -typedef struct { - int level; - int did; -} SDiskID; - -typedef struct { - uint64_t size; - uint64_t free; - uint64_t nfiles; -} SDiskMeta; - -typedef struct { - uint64_t tsize; - uint64_t avail; // bytes -} STiersMeta; - -typedef struct { - int level; - int did; - char dir[TSDB_FILENAME_LEN]; - SDiskMeta dmeta; -} SDisk; - -typedef struct { - int level; - int nDisks; - SDisk *disks[TSDB_MAX_DISKS_PER_TIER]; -} STier; - -typedef struct SDnodeTier { - pthread_mutex_t lock; - STiersMeta meta; - int nTiers; - STier tiers[TSDB_MAX_TIERS]; - SHashObj * map; -} SDnodeTier; - -extern struct SDnodeTier *tsDnodeTier; -#define DNODE_PRIMARY_DISK(pDnodeTier) (pDnodeTier)->tiers[0].disks[0] - -static FORCE_INLINE int tdLockTiers(SDnodeTier *pDnodeTier) { - int code = pthread_mutex_lock(&(pDnodeTier->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -static FORCE_INLINE int tdUnLockTiers(SDnodeTier *pDnodeTier) { - int code = pthread_mutex_unlock(&(pDnodeTier->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -static FORCE_INLINE SDisk *tdGetDisk(SDnodeTier *pDnodeTier, int level, int did) { - if (level < 0 || level >= pDnodeTier->nTiers) return NULL; - - if (did < 0 || did >= pDnodeTier->tiers[level].nDisks) return NULL; - - return pDnodeTier->tiers[level].disks[did]; -} - -SDnodeTier *tdNewTier(); -void * tdCloseTier(SDnodeTier *pDnodeTier); -int tdAddDisks(SDnodeTier *pDnodeTier, SDiskCfg *pDiskCfgs, int ndisks); -int tdUpdateTiersInfo(SDnodeTier *pDnodeTier); -int tdCheckTiers(SDnodeTier *pDnodeTier); -SDisk * tdAssignDisk(SDnodeTier *pDnodeTier, int level); -SDisk * tdGetDiskByName(SDnodeTier *pDnodeTier, char *dirName); -void tdIncDiskFiles(SDnodeTier *pDnodeTier, SDisk *pDisk, bool lock); -void tdDecDiskFiles(SDnodeTier *pDnodeTier, SDisk *pDisk, bool lock); +int tdInitTiers(SDiskCfg *pDiskCfg, int ndisk); +void tdDestroyTiers(); +int tdUpdateDiskInfos(); +void tdGetPrimaryPath(char *dst); #ifdef __cplusplus } diff --git a/src/common/src/tdisk.c b/src/common/src/tdisk.c index 63efc67aa6..2478b4ddd9 100644 --- a/src/common/src/tdisk.c +++ b/src/common/src/tdisk.c @@ -17,111 +17,139 @@ #include "tdisk.h" #include "tulog.h" +typedef struct { + int level; + int did; +} SDiskID; + +typedef struct { + uint64_t size; + uint64_t free; + uint64_t nfiles; +} SDiskMeta; + +typedef struct { + uint64_t tsize; + uint64_t avail; // bytes +} STiersMeta; + +typedef struct { + int level; + int did; + char dir[TSDB_FILENAME_LEN]; + SDiskMeta dmeta; +} SDisk; + +typedef struct { + int level; + int nDisks; + SDisk *disks[TSDB_MAX_DISKS_PER_TIER]; +} STier; + +typedef struct STiers { + pthread_mutex_t lock; + STiersMeta meta; + int nLevel; + STier tiers[TSDB_MAX_TIERS]; + SHashObj * map; +} STiers; + #define DISK_MIN_FREE_SPACE 30 * 1024 * 1024 // disk free space less than 100M will not create new file again #define DNODE_DISK_AVAIL(pDisk) ((pDisk)->dmeta.free > DISK_MIN_FREE_SPACE) +#define TIER_AT_LEVEL(level) (pTiers->tiers + (level)) +#define DISK_AT(level, did) (TIER_AT_LEVEL(level)->disks[(did)]) -static int tdFormatDir(char *idir, char *odir); -static int tdCheckDisk(char *dirName, int level, int primary); -static int tdUpdateDiskMeta(SDisk *pDisk); -static int tdAddDisk(SDnodeTier *pDnodeTier, char *dir, int level, int primary); +static struct STiers tdTiers; +static struct STiers *pTiers = &tdTiers; -struct SDnodeTier *tsDnodeTier = NULL; +int tdInitTiers(SDiskCfg *pDiskCfg, int ndisk) { + ASSERT(ndisk > 0); -SDnodeTier *tdNewTier() { - SDnodeTier *pDnodeTier = (SDnodeTier *)calloc(1, sizeof(*pDnodeTier)); - if (pDnodeTier == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; - } + memset((void *)pTiers, 0, sizeof(*pTiers)); - int ret = pthread_mutex_init(&(pDnodeTier->lock), NULL); + int ret = pthread_mutex_init(&(pTiers->lock), NULL); if (ret != 0) { terrno = TAOS_SYSTEM_ERROR(ret); - tdCloseTier(pDnodeTier); - return NULL; + return -1; } - pDnodeTier->map = taosHashInit(TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER * 2, - taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (pDnodeTier->map == NULL) { + pTiers->map = taosHashInit(TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER * 2, + taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (pTiers->map == NULL) { terrno = TSDB_CODE_COM_OUT_OF_MEMORY; - tdCloseTier(pDnodeTier); - return NULL; + tdDestroyTiers(); + return -1; } - return pDnodeTier; -} - -void *tdCloseTier(SDnodeTier *pDnodeTier) { - if (pDnodeTier) { - if (pDnodeTier->map) { - taosHashCleanup(pDnodeTier->map); - pDnodeTier->map = NULL; + for (int idisk = 0; idisk < ndisk; idisk++) { + if (tdAddDisk(pDiskCfg + idisk) < 0) { + tdDestroyTiers(); + return -1; } - - pthread_mutex_destroy(&(pDnodeTier->lock)); - - for (int i = 0; i < pDnodeTier->nTiers; i++) { - STier *pTier = pDnodeTier->tiers + i; - for (int j = 0; j < pTier->nDisks; j++) { - if (pTier->disks[j]) { - free(pTier->disks[j]); - pTier->disks[j] = NULL; - } - } - } - free(pDnodeTier); - } - return NULL; -} - -int tdAddDisks(SDnodeTier *pDnodeTier, SDiskCfg *pDiskCfgs, int ndisks) { - ASSERT(ndisks > 0); - - for (int i = 0; i < ndisks; i++) { - SDiskCfg *pCfg = pDiskCfgs + i; - tdAddDisk(pDnodeTier, pCfg->dir, pCfg->level, pCfg->primary); } - if (tdCheckTiers(pDnodeTier) < 0) return -1; - + if (tdCheckTiers() < 0) { + tdDestroyTiers(); + return -1; + } + return 0; } -int tdUpdateTiersInfo(SDnodeTier *pDnodeTier) { - tdLockTiers(pDnodeTier); +void tdDestroyTiers() { + taosHashCleanup(pTiers->map); + pTiers->map = NULL; - pDnodeTier->meta.tsize = 0; - pDnodeTier->meta.avail = 0; + pthread_mutex_destroy(&(pTiers->lock)); - for (int i = 0; i < pDnodeTier->nTiers; i++) { - STier *pTier = pDnodeTier->tiers + i; + for (int level = TSDB_MAX_TIERS - 1; level >= 0; --level) { + for (int did = TSDB_MAX_DISKS_PER_TIER - 1; did >= 0; --did) { + if (DISK_AT(level, did)) { + free(DISK_AT(level, did)); + DISK_AT(level, did) = NULL; + } + } + } +} + +int tdUpdateDiskInfos() { + tdLockTiers(pTiers); + + pTiers->meta.tsize = 0; + pTiers->meta.avail = 0; + + for (int i = 0; i < pTiers->nLevel; i++) { + STier *pTier = pTiers->tiers + i; for (int j = 0; j < pTier->nDisks; j++) { SDisk *pDisk = pTier->disks[j]; if (tdUpdateDiskMeta(pDisk) < 0) { - tdUnLockTiers(pDnodeTier); + tdUnLockTiers(pTiers); return -1; } - pDnodeTier->meta.tsize += pDisk->dmeta.size; - pDnodeTier->meta.avail += pDisk->dmeta.free; + pTiers->meta.tsize += pDisk->dmeta.size; + pTiers->meta.avail += pDisk->dmeta.free; } } - tdUnLockTiers(pDnodeTier); + tdUnLockTiers(pTiers); return 0; } -int tdCheckTiers(SDnodeTier *pDnodeTier) { - ASSERT(pDnodeTier->nTiers > 0); - if (DNODE_PRIMARY_DISK(pDnodeTier) == NULL) { +void tdGetPrimaryPath(char *dst) { strncpy(dst, DISK_AT(0, 0)->dir, TSDB_FILENAME_LEN); } + +static SDisk *tdGetPrimaryDisk() { return DISK_AT(0, 0); } + +static int tdCheckTiers() { + ASSERT(pTiers->nLevel > 0); + if (tdGetPrimaryDisk(pTiers) == NULL) { terrno = TSDB_CODE_DND_LACK_PRIMARY_DISK; return -1; } - for (int i = 0; i < pDnodeTier->nTiers; i++) { - if (pDnodeTier->tiers[i].nDisks == 0) { + for (int i = 0; i < pTiers->nLevel; i++) { + if (pTiers->tiers[i].nDisks == 0) { terrno = TSDB_CODE_DND_NO_DISK_AT_TIER; return -1; } @@ -130,15 +158,15 @@ int tdCheckTiers(SDnodeTier *pDnodeTier) { return 0; } -SDisk *tdAssignDisk(SDnodeTier *pDnodeTier, int level) { - ASSERT(level < pDnodeTier->nTiers); +static SDisk *tdAssignDisk(int level) { + ASSERT(level < pTiers->nLevel); - STier *pTier = pDnodeTier->tiers + level; + STier *pTier = pTiers->tiers + level; SDisk *pDisk = NULL; ASSERT(pTier->nDisks > 0); - tdLockTiers(pDnodeTier); + tdLockTiers(pTiers); for (int i = 0; i < pTier->nDisks; i++) { SDisk *iDisk = pTier->disks[i]; @@ -152,18 +180,18 @@ SDisk *tdAssignDisk(SDnodeTier *pDnodeTier, int level) { if (pDisk == NULL) { terrno = TSDB_CODE_DND_NO_DISK_SPACE; - tdUnLockTiers(pDnodeTier); + tdUnLockTiers(pTiers); return NULL; } - tdIncDiskFiles(pDnodeTier, pDisk, false); + tdIncDiskFiles(pTiers, pDisk, false); - tdUnLockTiers(pDnodeTier); + tdUnLockTiers(pTiers); return pDisk; } -SDisk *tdGetDiskByName(SDnodeTier *pDnodeTier, char *dirName) { +static SDisk *tdGetDiskByName(char *dirName) { char fdirName[TSDB_FILENAME_LEN] = "\0"; SDiskID *pDiskID = NULL; @@ -171,34 +199,34 @@ SDisk *tdGetDiskByName(SDnodeTier *pDnodeTier, char *dirName) { return NULL; } - void *ptr = taosHashGet(pDnodeTier->map, (void *)fdirName, strnlen(fdirName, TSDB_FILENAME_LEN)); + void *ptr = taosHashGet(pTiers->map, (void *)fdirName, strnlen(fdirName, TSDB_FILENAME_LEN)); if (ptr == NULL) return NULL; pDiskID = (SDiskID *)ptr; - return tdGetDisk(pDnodeTier, pDiskID->level, pDiskID->did); + return tdGetDisk(pTiers, pDiskID->level, pDiskID->did); } -void tdIncDiskFiles(SDnodeTier *pDnodeTier, SDisk *pDisk, bool lock) { +static void tdIncDiskFiles(SDisk *pDisk, bool lock) { if (lock) { - tdLockTiers(pDnodeTier); + tdLockTiers(pTiers); } pDisk->dmeta.nfiles++; if (lock) { - tdUnLockTiers(pDnodeTier); + tdUnLockTiers(pTiers); } } -void tdDecDiskFiles(SDnodeTier *pDnodeTier, SDisk *pDisk, bool lock) { +static void tdDecDiskFiles(SDisk *pDisk, bool lock) { if (lock) { - tdLockTiers(pDnodeTier); + tdLockTiers(pTiers); } pDisk->dmeta.nfiles--; if (lock) { - tdUnLockTiers(pDnodeTier); + tdUnLockTiers(pTiers); } } @@ -223,7 +251,7 @@ static int tdFormatDir(char *idir, char *odir) { return 0; } -static int tdCheckDisk(char *dirName, int level, int primary) { +static int tdCheckDisk(char *dirName) { if (access(dirName, W_OK | R_OK | F_OK) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -257,66 +285,66 @@ static int tdUpdateDiskMeta(SDisk *pDisk) { return 0; } -static int tdAddDisk(SDnodeTier *pDnodeTier, char *dir, int level, int primary) { +static int tdAddDisk(SDiskCfg *pCfg) { char dirName[TSDB_FILENAME_LEN] = "\0"; STier * pTier = NULL; SDiskID diskid = {0}; SDisk * pDisk = NULL; - if (level < 0 || level >= TSDB_MAX_TIERS) { + if (pCfg->level < 0 || pCfg->level >= TSDB_MAX_TIERS) { terrno = TSDB_CODE_DND_INVALID_DISK_TIER; - uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno)); + uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno)); return -1; } - if (tdFormatDir(dir, dirName) < 0) { - uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno)); + if (tdFormatDir(pCfg->dir, dirName) < 0) { + uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno)); return -1; } - pTier = pDnodeTier->tiers + level; - diskid.level = level; + pTier = TIER_AT_LEVEL(pCfg->level); + diskid.level = pCfg->level; if (pTier->nDisks >= TSDB_MAX_DISKS_PER_TIER) { terrno = TSDB_CODE_DND_TOO_MANY_DISKS; - uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno)); + uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno)); return -1; } - if (tdGetDiskByName(pDnodeTier, dirName) != NULL) { + if (tdGetDiskByName(dirName) != NULL) { terrno = TSDB_CODE_DND_DISK_ALREADY_EXISTS; - uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno)); + uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno)); return -1; } - if (tdCheckDisk(dirName, level, primary) < 0) { - uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno)); + if (tdCheckDisk(dirName) < 0) { + uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno)); return -1; } - if (primary) { - if (level != 0) { + if (pCfg->primary) { + if (pCfg->level != 0) { terrno = TSDB_CODE_DND_INVALID_DISK_TIER; - uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno)); + uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno)); return -1; } - if (DNODE_PRIMARY_DISK(pDnodeTier) != NULL) { + if (tdGetPrimaryDisk() != NULL) { terrno = TSDB_CODE_DND_DUPLICATE_PRIMARY_DISK; - uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno)); + uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno)); return -1; } diskid.did = 0; } else { if (level == 0) { - if (DNODE_PRIMARY_DISK(pDnodeTier) != NULL) { + if (tdGetPrimaryDisk() != NULL) { diskid.did = pTier->nDisks; } else { diskid.did = pTier->nDisks + 1; if (diskid.did >= TSDB_MAX_DISKS_PER_TIER) { terrno = TSDB_CODE_DND_TOO_MANY_DISKS; - uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno)); + uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno)); return -1; } } @@ -328,7 +356,7 @@ static int tdAddDisk(SDnodeTier *pDnodeTier, char *dir, int level, int primary) pDisk = (SDisk *)calloc(1, sizeof(SDisk)); if (pDisk == NULL) { terrno = TSDB_CODE_DND_OUT_OF_MEMORY; - uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno)); + uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno)); return -1; } @@ -336,27 +364,27 @@ static int tdAddDisk(SDnodeTier *pDnodeTier, char *dir, int level, int primary) pDisk->level = diskid.level; pDisk->did = diskid.did; - if (taosHashPut(pDnodeTier->map, (void *)dirName, strnlen(dirName, TSDB_FILENAME_LEN), (void *)(&diskid), + if (taosHashPut(pTiers->map, (void *)dirName, strnlen(dirName, TSDB_FILENAME_LEN), (void *)(&diskid), sizeof(diskid)) < 0) { free(pDisk); terrno = TSDB_CODE_DND_OUT_OF_MEMORY; - uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno)); + uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno)); return -1; } pTier->nDisks++; pTier->disks[diskid.did] = pDisk; - pDnodeTier->nTiers = MAX(pDnodeTier->nTiers, level + 1); + pTiers->nLevel = MAX(pTiers->nLevel, level + 1); return 0; } -void taosGetDisk() { +static void taosGetDisk() { const double unit = 1024 * 1024 * 1024; SysDiskSize diskSize; if (tscEmbedded) { - tdUpdateTiersInfo(tsDnodeTier); + tdUpdateDiskInfos(tsDnodeTier); tsTotalDataDirGB = (float)tsDnodeTier->meta.tsize / unit; tsAvailDataDirGB = (float)tsDnodeTier->meta.avail / unit; } @@ -371,3 +399,29 @@ void taosGetDisk() { tsAvailTmpDirectorySpace = (float)diskSize.avail / unit; } } + +static int tdLockTiers() { + int code = pthread_mutex_lock(&(pTiers->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +static int tdUnLockTiers() { + int code = pthread_mutex_unlock(&(pTiers->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +static SDisk *tdGetDisk(int level, int did) { + if (level < 0 || level >= pTiers->nLevel) return NULL; + + if (did < 0 || did >= pTiers->tiers[level].nDisks) return NULL; + + return pTiers->tiers[level].disks[did]; +} \ No newline at end of file diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index c4c5dcc911..5e4136bd8c 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -183,17 +183,11 @@ static void dnodeCheckDataDirOpenned(char *dir) { } static int32_t dnodeInitStorage() { - tsDnodeTier = tdNewTier(); - if (tsDnodeTier == NULL) { - dError("failed to create new dnode tier since %s", tstrerror(terrno)); - return -1; - } - - if (tdAddDisks(tsDnodeTier, tsDiskCfg, tsDiskCfgNum) < 0) { + if (tdInitTiers(tsDiskCfg, tsDiskCfgNum) < 0) { dError("failed to add disks to dnode tier since %s", tstrerror(terrno)); return -1; } - strncpy(tsDataDir, DNODE_PRIMARY_DISK(tsDnodeTier)->dir, TSDB_FILENAME_LEN); + tdGetPrimaryPath(tsDataDir); tdGetVnodeRootDir(tsDataDir, tsVnodeDir); //TODO(dengyihao): no need to init here @@ -236,12 +230,7 @@ static int32_t dnodeInitStorage() { return 0; } -static void dnodeCleanupStorage() { - if (tsDnodeTier) { - tdCloseTier(tsDnodeTier); - tsDnodeTier = NULL; - } -} +static void dnodeCleanupStorage() { tdDestroyTiers(); } bool dnodeIsFirstDeploy() { return strcmp(tsFirst, tsLocalEp) == 0;