enh: check if disk space sufficient at primary dir with tfs

include/libs/tfs/tfs.h

@@ -300,6 +300,25 @@ void tfsClosedir(STfsDir *pDir);
  */
 int32_t tfsGetMonitorInfo(STfs *pTfs, SMonDiskInfo *pInfo);
 
+/**
+ * @brief Check if disk space available at level
+ *
+ * @param pTfs The fs object.
+ * #param level the level
+ * @return bool
+ */
+bool tfsDiskSpaceAvailable(STfs *pTfs, int32_t level);
+
+/**
+ * @brief Check if disk space sufficient at disk of level
+ *
+ * @param pTfs The fs object.
+ * @param level the level
+ * @param disk the disk
+ * @return bool
+ */
+bool tfsDiskSpaceSufficient(STfs *pTfs, int32_t level, int32_t disk);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/mgmt/mgmt_vnode/src/vmInt.c

@@ -460,7 +460,6 @@ static void vmCleanup(SVnodeMgmt *pMgmt) {
   vmCloseVnodes(pMgmt);
   vmStopWorker(pMgmt);
   vnodeCleanup();
-  tfsClose(pMgmt->pTfs);
   taosThreadRwlockDestroy(&pMgmt->lock);
   taosMemoryFree(pMgmt);
 }
@@ -535,20 +534,9 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
   pMgmt->msgCb.mgmt = pMgmt;
   taosThreadRwlockInit(&pMgmt->lock, NULL);
 
-  SDiskCfg dCfg = {0};
-  tstrncpy(dCfg.dir, tsDataDir, TSDB_FILENAME_LEN);
-  dCfg.level = 0;
-  dCfg.primary = 1;
-  SDiskCfg *pDisks = tsDiskCfg;
-  int32_t   numOfDisks = tsDiskCfgNum;
-  if (numOfDisks <= 0 || pDisks == NULL) {
-    pDisks = &dCfg;
-    numOfDisks = 1;
-  }
-
-  pMgmt->pTfs = tfsOpen(pDisks, numOfDisks);
+  pMgmt->pTfs = pInput->pTfs;
   if (pMgmt->pTfs == NULL) {
-    dError("failed to init tfs since %s", terrstr());
+    dError("tfs is null.");
     goto _OVER;
   }
   tmsgReportStartup("vnode-tfs", "initialized");

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -15,6 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "vmInt.h"
+#include "vnodeInt.h"
 
 static inline void vmSendRsp(SRpcMsg *pMsg, int32_t code) {
   if (pMsg->info.handle == NULL) return;
@@ -158,6 +159,15 @@ static void vmSendResponse(SRpcMsg *pMsg) {
   }
 }
 
+static bool vmDataSpaceSufficient(SVnodeObj *pVnode) {
+  STfs *pTfs = pVnode->pImpl->pTfs;
+  if (pTfs) {
+    return tfsDiskSpaceSufficient(pTfs, 0, pVnode->diskPrimary);
+  } else {
+    return osDataSpaceSufficient();
+  }
+}
+
 static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtype) {
   const STraceId *trace = &pMsg->info.traceId;
   if (pMsg->contLen < sizeof(SMsgHead)) {
@@ -203,7 +213,7 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
       taosWriteQitem(pVnode->pFetchQ, pMsg);
       break;
     case WRITE_QUEUE:
-      if (!osDataSpaceSufficient()) {
+      if (!vmDataSpaceSufficient(pVnode)) {
         terrno = TSDB_CODE_NO_ENOUGH_DISKSPACE;
         code = terrno;
         dError("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, terrstr(code));

source/dnode/mgmt/node_mgmt/inc/dmMgmt.h

@@ -20,6 +20,7 @@
 #include "uv.h"
 
 #include "dmInt.h"
+#include "tfs.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -79,6 +80,7 @@ typedef struct SDnode {
   TdThreadMutex mutex;
   TdFilePtr     lockfile;
   SDnodeData    data;
+  STfs         *pTfs;
   SMgmtWrapper  wrappers[NODE_END];
 } SDnode;
 
@@ -124,4 +126,4 @@ void dmGetQnodeLoads(SQnodeLoad *pInfo);
 }
 #endif
 
-#endif /*_TD_DND_MGMT_H_*/
+#endif /*_TD_DND_MGMT_H_*/

source/dnode/mgmt/node_mgmt/src/dmEnv.c

@@ -96,28 +96,23 @@ _exit:
   return code;
 }
 
-static bool dmCheckDiskSpace() {
-  osUpdate();
-  // sufficiency
-  if (!osDataSpaceSufficient()) {
-    dWarn("free data disk size: %f GB, not sufficient, expected %f GB at least",
-          (double)tsDataSpace.size.avail / 1024.0 / 1024.0 / 1024.0,
-          (double)tsDataSpace.reserved / 1024.0 / 1024.0 / 1024.0);
+static bool dmDataSpaceAvailable() {
+  SDnode *pDnode = dmInstance();
+  if (pDnode->pTfs) {
+    return tfsDiskSpaceAvailable(pDnode->pTfs, 0);
   }
-  if (!osLogSpaceSufficient()) {
-    dWarn("free log disk size: %f GB, not sufficient, expected %f GB at least",
-          (double)tsLogSpace.size.avail / 1024.0 / 1024.0 / 1024.0,
-          (double)tsLogSpace.reserved / 1024.0 / 1024.0 / 1024.0);
-  }
-  if (!osTempSpaceSufficient()) {
-    dWarn("free temp disk size: %f GB, not sufficient, expected %f GB at least",
-          (double)tsTempSpace.size.avail / 1024.0 / 1024.0 / 1024.0,
-          (double)tsTempSpace.reserved / 1024.0 / 1024.0 / 1024.0);
-  }
-  // availability
-  bool ret = true;
   if (!osDataSpaceAvailable()) {
     dError("data disk space unavailable, i.e. %s", tsDataDir);
+    return false;
+  }
+  return true;
+}
+
+static bool dmCheckDiskSpace() {
+  osUpdate();
+  // availability
+  bool ret = true;
+  if (!dmDataSpaceAvailable()) {
     terrno = TSDB_CODE_NO_DISKSPACE;
     ret = false;
   }
@@ -134,6 +129,34 @@ static bool dmCheckDiskSpace() {
   return ret;
 }
 
+int32_t dmDiskInit() {
+  SDnode  *pDnode = dmInstance();
+  SDiskCfg dCfg = {0};
+  tstrncpy(dCfg.dir, tsDataDir, TSDB_FILENAME_LEN);
+  dCfg.level = 0;
+  dCfg.primary = 1;
+  SDiskCfg *pDisks = tsDiskCfg;
+  int32_t   numOfDisks = tsDiskCfgNum;
+  if (numOfDisks <= 0 || pDisks == NULL) {
+    pDisks = &dCfg;
+    numOfDisks = 1;
+  }
+
+  pDnode->pTfs = tfsOpen(pDisks, numOfDisks);
+  if (pDnode->pTfs == NULL) {
+    dError("failed to init tfs since %s", terrstr());
+    return -1;
+  }
+  return 0;
+}
+
+int32_t dmDiskClose() {
+  SDnode *pDnode = dmInstance();
+  tfsClose(pDnode->pTfs);
+  pDnode->pTfs = NULL;
+  return 0;
+}
+
 static bool dmCheckDataDirVersion() {
   char checkDataDirJsonFileName[PATH_MAX] = {0};
   snprintf(checkDataDirJsonFileName, PATH_MAX, "%s/dnode/dnodeCfg.json", tsDataDir);
@@ -147,6 +170,7 @@ static bool dmCheckDataDirVersion() {
 
 int32_t dmInit() {
   dInfo("start to init dnode env");
+  if (dmDiskInit() != 0) return -1;
   if (!dmCheckDataDirVersion()) return -1;
   if (!dmCheckDiskSpace()) return -1;
   if (dmCheckRepeatInit(dmInstance()) != 0) return -1;
@@ -177,6 +201,7 @@ void dmCleanup() {
   udfcClose();
   udfStopUdfd();
   taosStopCacheRefreshWorker();
+  dmDiskClose();
   dInfo("dnode env is cleaned up");
 
   taosCleanupCfg();
@@ -367,6 +392,7 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) {
   SMgmtInputOpt opt = {
       .path = pWrapper->path,
       .name = pWrapper->name,
+      .pTfs = pWrapper->pDnode->pTfs,
       .pData = &pWrapper->pDnode->data,
       .processCreateNodeFp = dmProcessCreateNodeReq,
       .processAlterNodeTypeFp = dmProcessAlterNodeTypeReq,

source/dnode/mgmt/node_util/inc/dmUtil.h

@@ -37,6 +37,7 @@
 #include "monitor.h"
 #include "qnode.h"
 #include "sync.h"
+#include "tfs.h"
 #include "wal.h"
 
 #include "libs/function/tudf.h"
@@ -111,6 +112,7 @@ typedef struct {
 typedef struct {
   const char         *path;
   const char         *name;
+  STfs               *pTfs;
   SDnodeData         *pData;
   SMsgCb              msgCb;
   ProcessCreateNodeFp processCreateNodeFp;

source/libs/tfs/src/tfs.c

@@ -14,6 +14,7 @@
  */
 
 #define _DEFAULT_SOURCE
+#include "osEnv.h"
 #include "tfsInt.h"
 
 static int32_t   tfsMount(STfs *pTfs, SDiskCfg *pCfg);
@@ -113,6 +114,39 @@ SDiskSize tfsGetSize(STfs *pTfs) {
   return size;
 }
 
+bool tfsDiskSpaceAvailable(STfs *pTfs, int32_t level) {
+  if (level < 0 || level >= pTfs->nlevel) {
+    return false;
+  }
+  STfsTier *pTier = TFS_TIER_AT(pTfs, level);
+  for (int32_t id = 0; id < pTier->ndisk; id++) {
+    SDiskID   diskId = {.level = level, .id = id};
+    STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId);
+    if (pDisk == NULL) {
+      return false;
+    }
+    if (pDisk->size.avail <= 0) {
+      fError("tfs disk space unavailable. level:%d, disk:%d, path:%s", level, id, pDisk->path);
+      return false;
+    }
+  }
+  return true;
+}
+
+bool tfsDiskSpaceSufficient(STfs *pTfs, int32_t level, int32_t disk) {
+  if (level < 0 || level >= pTfs->nlevel) {
+    return false;
+  }
+
+  STfsTier *pTier = TFS_TIER_AT(pTfs, level);
+  if (disk < 0 || disk >= pTier->ndisk) {
+    return false;
+  }
+  SDiskID   diskId = {.level = level, .id = disk};
+  STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId);
+  return pDisk->size.avail >= tsDataSpace.reserved;
+}
+
 int32_t tfsGetDisksAtLevel(STfs *pTfs, int32_t level) {
   if (level < 0 || level >= pTfs->nlevel) {
     return 0;

source/os/src/osEnv.c

@@ -95,10 +95,10 @@ void osCleanup() {}
 
 bool osLogSpaceAvailable() { return tsLogSpace.size.avail > 0; }
 
-bool osDataSpaceAvailable() { return tsDataSpace.size.avail > 0; }
-
 bool osTempSpaceAvailable() { return tsTempSpace.size.avail > 0; }
 
+bool osDataSpaceAvailable() { return tsDataSpace.size.avail > 0; }
+
 bool osLogSpaceSufficient() { return tsLogSpace.size.avail > tsLogSpace.reserved; }
 
 bool osDataSpaceSufficient() { return tsDataSpace.size.avail > tsDataSpace.reserved; }