From e4d16e594cd1f78e3a8938aba13d1b9b787a9947 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 8 Aug 2023 17:46:37 +0800 Subject: [PATCH] enh: check if disk space sufficient at primary dir with tfs --- include/libs/tfs/tfs.h | 19 ++++++ source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 16 +----- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 12 +++- source/dnode/mgmt/node_mgmt/inc/dmMgmt.h | 4 +- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 64 +++++++++++++++------ source/dnode/mgmt/node_util/inc/dmUtil.h | 2 + source/libs/tfs/src/tfs.c | 34 +++++++++++ source/os/src/osEnv.c | 4 +- 8 files changed, 118 insertions(+), 37 deletions(-) diff --git a/include/libs/tfs/tfs.h b/include/libs/tfs/tfs.h index 509f8dc9e8..2b90e3226c 100644 --- a/include/libs/tfs/tfs.h +++ b/include/libs/tfs/tfs.h @@ -300,6 +300,25 @@ void tfsClosedir(STfsDir *pDir); */ int32_t tfsGetMonitorInfo(STfs *pTfs, SMonDiskInfo *pInfo); +/** + * @brief Check if disk space available at level + * + * @param pTfs The fs object. + * #param level the level + * @return bool + */ +bool tfsDiskSpaceAvailable(STfs *pTfs, int32_t level); + +/** + * @brief Check if disk space sufficient at disk of level + * + * @param pTfs The fs object. + * @param level the level + * @param disk the disk + * @return bool + */ +bool tfsDiskSpaceSufficient(STfs *pTfs, int32_t level, int32_t disk); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 94a753062c..0ff2537e4c 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -460,7 +460,6 @@ static void vmCleanup(SVnodeMgmt *pMgmt) { vmCloseVnodes(pMgmt); vmStopWorker(pMgmt); vnodeCleanup(); - tfsClose(pMgmt->pTfs); taosThreadRwlockDestroy(&pMgmt->lock); taosMemoryFree(pMgmt); } @@ -535,20 +534,9 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->msgCb.mgmt = pMgmt; taosThreadRwlockInit(&pMgmt->lock, NULL); - SDiskCfg dCfg = {0}; - tstrncpy(dCfg.dir, tsDataDir, TSDB_FILENAME_LEN); - dCfg.level = 0; - dCfg.primary = 1; - SDiskCfg *pDisks = tsDiskCfg; - int32_t numOfDisks = tsDiskCfgNum; - if (numOfDisks <= 0 || pDisks == NULL) { - pDisks = &dCfg; - numOfDisks = 1; - } - - pMgmt->pTfs = tfsOpen(pDisks, numOfDisks); + pMgmt->pTfs = pInput->pTfs; if (pMgmt->pTfs == NULL) { - dError("failed to init tfs since %s", terrstr()); + dError("tfs is null."); goto _OVER; } tmsgReportStartup("vnode-tfs", "initialized"); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 247c1729a3..d567f1128e 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "vmInt.h" +#include "vnodeInt.h" static inline void vmSendRsp(SRpcMsg *pMsg, int32_t code) { if (pMsg->info.handle == NULL) return; @@ -158,6 +159,15 @@ static void vmSendResponse(SRpcMsg *pMsg) { } } +static bool vmDataSpaceSufficient(SVnodeObj *pVnode) { + STfs *pTfs = pVnode->pImpl->pTfs; + if (pTfs) { + return tfsDiskSpaceSufficient(pTfs, 0, pVnode->diskPrimary); + } else { + return osDataSpaceSufficient(); + } +} + static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtype) { const STraceId *trace = &pMsg->info.traceId; if (pMsg->contLen < sizeof(SMsgHead)) { @@ -203,7 +213,7 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp taosWriteQitem(pVnode->pFetchQ, pMsg); break; case WRITE_QUEUE: - if (!osDataSpaceSufficient()) { + if (!vmDataSpaceSufficient(pVnode)) { terrno = TSDB_CODE_NO_ENOUGH_DISKSPACE; code = terrno; dError("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, terrstr(code)); diff --git a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h index 02cd678433..98489433b9 100644 --- a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h +++ b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h @@ -20,6 +20,7 @@ #include "uv.h" #include "dmInt.h" +#include "tfs.h" #ifdef __cplusplus extern "C" { @@ -79,6 +80,7 @@ typedef struct SDnode { TdThreadMutex mutex; TdFilePtr lockfile; SDnodeData data; + STfs *pTfs; SMgmtWrapper wrappers[NODE_END]; } SDnode; @@ -124,4 +126,4 @@ void dmGetQnodeLoads(SQnodeLoad *pInfo); } #endif -#endif /*_TD_DND_MGMT_H_*/ \ No newline at end of file +#endif /*_TD_DND_MGMT_H_*/ diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 3f9c5bbeaf..a34002161d 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -96,28 +96,23 @@ _exit: return code; } -static bool dmCheckDiskSpace() { - osUpdate(); - // sufficiency - if (!osDataSpaceSufficient()) { - dWarn("free data disk size: %f GB, not sufficient, expected %f GB at least", - (double)tsDataSpace.size.avail / 1024.0 / 1024.0 / 1024.0, - (double)tsDataSpace.reserved / 1024.0 / 1024.0 / 1024.0); +static bool dmDataSpaceAvailable() { + SDnode *pDnode = dmInstance(); + if (pDnode->pTfs) { + return tfsDiskSpaceAvailable(pDnode->pTfs, 0); } - if (!osLogSpaceSufficient()) { - dWarn("free log disk size: %f GB, not sufficient, expected %f GB at least", - (double)tsLogSpace.size.avail / 1024.0 / 1024.0 / 1024.0, - (double)tsLogSpace.reserved / 1024.0 / 1024.0 / 1024.0); - } - if (!osTempSpaceSufficient()) { - dWarn("free temp disk size: %f GB, not sufficient, expected %f GB at least", - (double)tsTempSpace.size.avail / 1024.0 / 1024.0 / 1024.0, - (double)tsTempSpace.reserved / 1024.0 / 1024.0 / 1024.0); - } - // availability - bool ret = true; if (!osDataSpaceAvailable()) { dError("data disk space unavailable, i.e. %s", tsDataDir); + return false; + } + return true; +} + +static bool dmCheckDiskSpace() { + osUpdate(); + // availability + bool ret = true; + if (!dmDataSpaceAvailable()) { terrno = TSDB_CODE_NO_DISKSPACE; ret = false; } @@ -134,6 +129,34 @@ static bool dmCheckDiskSpace() { return ret; } +int32_t dmDiskInit() { + SDnode *pDnode = dmInstance(); + SDiskCfg dCfg = {0}; + tstrncpy(dCfg.dir, tsDataDir, TSDB_FILENAME_LEN); + dCfg.level = 0; + dCfg.primary = 1; + SDiskCfg *pDisks = tsDiskCfg; + int32_t numOfDisks = tsDiskCfgNum; + if (numOfDisks <= 0 || pDisks == NULL) { + pDisks = &dCfg; + numOfDisks = 1; + } + + pDnode->pTfs = tfsOpen(pDisks, numOfDisks); + if (pDnode->pTfs == NULL) { + dError("failed to init tfs since %s", terrstr()); + return -1; + } + return 0; +} + +int32_t dmDiskClose() { + SDnode *pDnode = dmInstance(); + tfsClose(pDnode->pTfs); + pDnode->pTfs = NULL; + return 0; +} + static bool dmCheckDataDirVersion() { char checkDataDirJsonFileName[PATH_MAX] = {0}; snprintf(checkDataDirJsonFileName, PATH_MAX, "%s/dnode/dnodeCfg.json", tsDataDir); @@ -147,6 +170,7 @@ static bool dmCheckDataDirVersion() { int32_t dmInit() { dInfo("start to init dnode env"); + if (dmDiskInit() != 0) return -1; if (!dmCheckDataDirVersion()) return -1; if (!dmCheckDiskSpace()) return -1; if (dmCheckRepeatInit(dmInstance()) != 0) return -1; @@ -177,6 +201,7 @@ void dmCleanup() { udfcClose(); udfStopUdfd(); taosStopCacheRefreshWorker(); + dmDiskClose(); dInfo("dnode env is cleaned up"); taosCleanupCfg(); @@ -367,6 +392,7 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) { SMgmtInputOpt opt = { .path = pWrapper->path, .name = pWrapper->name, + .pTfs = pWrapper->pDnode->pTfs, .pData = &pWrapper->pDnode->data, .processCreateNodeFp = dmProcessCreateNodeReq, .processAlterNodeTypeFp = dmProcessAlterNodeTypeReq, diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 85057e5916..32c3d22506 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -37,6 +37,7 @@ #include "monitor.h" #include "qnode.h" #include "sync.h" +#include "tfs.h" #include "wal.h" #include "libs/function/tudf.h" @@ -111,6 +112,7 @@ typedef struct { typedef struct { const char *path; const char *name; + STfs *pTfs; SDnodeData *pData; SMsgCb msgCb; ProcessCreateNodeFp processCreateNodeFp; diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 8adaab91a1..445c24159f 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE +#include "osEnv.h" #include "tfsInt.h" static int32_t tfsMount(STfs *pTfs, SDiskCfg *pCfg); @@ -113,6 +114,39 @@ SDiskSize tfsGetSize(STfs *pTfs) { return size; } +bool tfsDiskSpaceAvailable(STfs *pTfs, int32_t level) { + if (level < 0 || level >= pTfs->nlevel) { + return false; + } + STfsTier *pTier = TFS_TIER_AT(pTfs, level); + for (int32_t id = 0; id < pTier->ndisk; id++) { + SDiskID diskId = {.level = level, .id = id}; + STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId); + if (pDisk == NULL) { + return false; + } + if (pDisk->size.avail <= 0) { + fError("tfs disk space unavailable. level:%d, disk:%d, path:%s", level, id, pDisk->path); + return false; + } + } + return true; +} + +bool tfsDiskSpaceSufficient(STfs *pTfs, int32_t level, int32_t disk) { + if (level < 0 || level >= pTfs->nlevel) { + return false; + } + + STfsTier *pTier = TFS_TIER_AT(pTfs, level); + if (disk < 0 || disk >= pTier->ndisk) { + return false; + } + SDiskID diskId = {.level = level, .id = disk}; + STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId); + return pDisk->size.avail >= tsDataSpace.reserved; +} + int32_t tfsGetDisksAtLevel(STfs *pTfs, int32_t level) { if (level < 0 || level >= pTfs->nlevel) { return 0; diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 7f0e6d1dee..0fc136c693 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -95,10 +95,10 @@ void osCleanup() {} bool osLogSpaceAvailable() { return tsLogSpace.size.avail > 0; } -bool osDataSpaceAvailable() { return tsDataSpace.size.avail > 0; } - bool osTempSpaceAvailable() { return tsTempSpace.size.avail > 0; } +bool osDataSpaceAvailable() { return tsDataSpace.size.avail > 0; } + bool osLogSpaceSufficient() { return tsLogSpace.size.avail > tsLogSpace.reserved; } bool osDataSpaceSufficient() { return tsDataSpace.size.avail > tsDataSpace.reserved; }