xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

support disk usage

This commit is contained in:
yihaoDeng 2024-11-06 15:25:50 +08:00
parent af80d0a6bf
commit a6436882a2
9 changed files with 125 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
include/common/tmsg.h
View File

@ -1748,6 +1748,18 @@ typedef struct {
int32_t learnerProgress; // use one reservered
} SVnodeLoad;
typedef struct {
int32_t vgId;
int64_t numOfTables;
int64_t memSize;
int64_t l1Size;
int64_t l2Size;
int64_t l3Size;
int64_t cacheSize;
int64_t walSize;
int64_t metaSize;
} SDbSizeStatisInfo;
typedef struct {
int32_t vgId;
int64_t nTimeSeries;

2
include/libs/executor/storageapi.h
View File

@ -279,7 +279,7 @@ typedef struct SStoreMeta {
int32_t (*getNumOfChildTables)(void* pVnode, int64_t uid, int64_t* numOfTables, int32_t* numOfCols);
void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables,
int64_t* numOfNormalTables);
int32_t (*getDBSize)(void* pVnode, int64_t* dataSize, int64_t* walSize, int64_t* metaSize);
int32_t (*getDBSize)(void* pVnode, SDbSizeStatisInfo* pInfo);
SMCtbCursor* (*openCtbCursor)(void* pVnode, tb_uid_t uid, int lock);
int32_t (*resumeCtbCursor)(SMCtbCursor* pCtbCur, int8_t first);

27
include/os/osDir.h
View File

@ -37,11 +37,11 @@ extern "C" {
#ifdef WINDOWS
#define TD_TMP_DIR_PATH "C:\\Windows\\Temp\\"
#define TD_TMP_DIR_PATH "C:\\Windows\\Temp\\"
#ifdef CUS_NAME
#define TD_CFG_DIR_PATH "C:\\"CUS_NAME"\\cfg\\"
#define TD_DATA_DIR_PATH "C:\\"CUS_NAME"\\data\\"
#define TD_LOG_DIR_PATH "C:\\"CUS_NAME"\\log\\"
#define TD_CFG_DIR_PATH "C:\\" CUS_NAME "\\cfg\\"
#define TD_DATA_DIR_PATH "C:\\" CUS_NAME "\\data\\"
#define TD_LOG_DIR_PATH "C:\\" CUS_NAME "\\log\\"
#else
#define TD_CFG_DIR_PATH "C:\\TDengine\\cfg\\"
#define TD_DATA_DIR_PATH "C:\\TDengine\\data\\"
@ -51,10 +51,10 @@ extern "C" {
#elif defined(_TD_DARWIN_64)
#ifdef CUS_PROMPT
#define TD_TMP_DIR_PATH "/tmp/"CUS_PROMPT"d/"
#define TD_CFG_DIR_PATH "/etc/"CUS_PROMPT"/"
#define TD_DATA_DIR_PATH "/var/lib/"CUS_PROMPT"/"
#define TD_LOG_DIR_PATH "/var/log/"CUS_PROMPT"/"
#define TD_TMP_DIR_PATH "/tmp/" CUS_PROMPT "d/"
#define TD_CFG_DIR_PATH "/etc/" CUS_PROMPT "/"
#define TD_DATA_DIR_PATH "/var/lib/" CUS_PROMPT "/"
#define TD_LOG_DIR_PATH "/var/log/" CUS_PROMPT "/"
#else
#define TD_TMP_DIR_PATH "/tmp/taosd/"
#define TD_CFG_DIR_PATH "/etc/taos/"
@ -64,16 +64,16 @@ extern "C" {
#else
#define TD_TMP_DIR_PATH "/tmp/"
#define TD_TMP_DIR_PATH "/tmp/"
#ifdef CUS_PROMPT
#define TD_CFG_DIR_PATH "/etc/"CUS_PROMPT"/"
#define TD_DATA_DIR_PATH "/var/lib/"CUS_PROMPT"/"
#define TD_LOG_DIR_PATH "/var/log/"CUS_PROMPT"/"
#define TD_CFG_DIR_PATH "/etc/" CUS_PROMPT "/"
#define TD_DATA_DIR_PATH "/var/lib/" CUS_PROMPT "/"
#define TD_LOG_DIR_PATH "/var/log/" CUS_PROMPT "/"
#else
#define TD_CFG_DIR_PATH "/etc/taos/"
#define TD_DATA_DIR_PATH "/var/lib/taos/"
#define TD_LOG_DIR_PATH "/var/log/taos/"
#endif // CUS_PROMPT
#endif // CUS_PROMPT
#endif
typedef struct TdDir *TdDirPtr;
@ -100,6 +100,7 @@ bool taosDirEntryIsDir(TdDirEntryPtr pDirEntry);
char *taosGetDirEntryName(TdDirEntryPtr pDirEntry);
int32_t taosCloseDir(TdDirPtr *ppDir);
int taosGetDirSize(const char *path, int64_t *size);
#ifdef __cplusplus
}
#endif

2
source/dnode/vnode/inc/vnode.h
View File

@ -80,7 +80,7 @@ int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list);
int32_t vnodeIsCatchUp(SVnode *pVnode);
ESyncRole vnodeGetRole(SVnode *pVnode);
int32_t vnodeGetArbToken(SVnode *pVnode, char *outToken);
int32_t vnodeGetDBSize(void *pVnode, int64_t *dataSize, int64_t *walSize, int64_t *metaSize);
int32_t vnodeGetDBSize(void *pVnode, SDbSizeStatisInfo *pInfo);
int32_t vnodeUpdateArbTerm(SVnode *pVnode, int64_t arbTerm);

24
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -84,16 +84,18 @@ typedef struct SQueryNode SQueryNode;
#define VNODE_META_TMP_DIR "meta.tmp"
#define VNODE_META_BACKUP_DIR "meta.backup"
#define VNODE_META_DIR "meta"
#define VNODE_TSDB_DIR "tsdb"
#define VNODE_TQ_DIR "tq"
#define VNODE_WAL_DIR "wal"
#define VNODE_TSMA_DIR "tsma"
#define VNODE_RSMA_DIR "rsma"
#define VNODE_RSMA0_DIR "tsdb"
#define VNODE_RSMA1_DIR "rsma1"
#define VNODE_RSMA2_DIR "rsma2"
#define VNODE_TQ_STREAM "stream"
#define VNODE_META_DIR "meta"
#define VNODE_TSDB_DIR "tsdb"
#define VNODE_TQ_DIR "tq"
#define VNODE_WAL_DIR "wal"
#define VNODE_TSMA_DIR "tsma"
#define VNODE_RSMA_DIR "rsma"
#define VNODE_RSMA0_DIR "tsdb"
#define VNODE_RSMA1_DIR "rsma1"
#define VNODE_RSMA2_DIR "rsma2"
#define VNODE_TQ_STREAM "stream"
#define VNODE_CACHE_DIR "cache.rdb"
#define VNODE_TSDB_CACHE_DIR VNODE_TSDB_DIR TD_DIRSEP VNODE_CACHE_DIR
#if SUSPEND_RESUME_TEST // only for test purpose
#define VNODE_BUFPOOL_SEGMENTS 1
@ -163,7 +165,7 @@ int32_t metaDropTables(SMeta* pMeta, SArray* tbUids);
int metaTtlFindExpired(SMeta* pMeta, int64_t timePointMs, SArray* tbUids, int32_t ttlDropMaxCount);
int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp);
int metaUpdateChangeTimeWithLock(SMeta* pMeta, tb_uid_t uid, int64_t changeTimeMs);
SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, int64_t *createTime);
SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, int64_t* createTime);
int32_t metaGetTbTSchemaNotNull(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, STSchema** ppTSchema);
int32_t metaGetTbTSchemaMaybeNull(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, STSchema** ppTSchema);
STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock);

19
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -870,7 +870,7 @@ int32_t vnodeGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64
return tsdbGetTableSchema(((SVnode *)pVnode)->pMeta, uid, pSchema, suid);
}
int32_t vnodeGetDBSize(void *pVnode, int64_t *dataSize, int64_t *walSize, int64_t *metaSize) {
int32_t vnodeGetDBSize(void *pVnode, SDbSizeStatisInfo *pInfo) {
SVnode *pVnodeObj = pVnode;
if (pVnodeObj == NULL) {
return TSDB_CODE_VND_NOT_EXIST;
@ -878,26 +878,27 @@ int32_t vnodeGetDBSize(void *pVnode, int64_t *dataSize, int64_t *walSize, int64_
int32_t code = 0;
char path[TSDB_FILENAME_LEN] = {0};
char *dirName[] = {VNODE_TSDB_DIR, VNODE_WAL_DIR, VNODE_META_DIR};
int64_t dirSize[3];
char *dirName[] = {VNODE_TSDB_DIR, VNODE_WAL_DIR, VNODE_META_DIR, VNODE_TSDB_CACHE_DIR};
int64_t dirSize[4];
vnodeGetPrimaryDir(pVnodeObj->path, pVnodeObj->diskPrimary, pVnodeObj->pTfs, path, TSDB_FILENAME_LEN);
int32_t offset = strlen(path);
for (int i = 0; i < sizeof(dirName) / sizeof(dirName[0]); i++) {
SDiskSize size = {0};
int64_t size = {0};
(void)snprintf(path + offset, TSDB_FILENAME_LEN, "%s%s", TD_DIRSEP, dirName[i]);
code = taosGetDiskSize(path, &size);
code = taosGetDirSize(path, &size);
if (code != 0) {
return code;
}
path[offset] = 0;
dirSize[i] = size.used;
dirSize[i] = size;
}
*dataSize = dirSize[0];
*walSize = dirSize[1];
*metaSize = dirSize[2];
pInfo->l1Size = dirSize[0];
pInfo->walSize = dirSize[1];
pInfo->metaSize = dirSize[2];
return 0;
}

24
source/libs/executor/src/sysscanoperator.c
View File

@ -2030,7 +2030,7 @@ static SSDataBlock* sysTableBuildVgUsage(SOperatorInfo* pOperator) {
setOperatorCompleted(pOperator);
return NULL;
return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes;
// return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes;
}
if (pInfo->pCur == NULL) {
pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode);
@ -2048,12 +2048,20 @@ static SSDataBlock* sysTableBuildVgUsage(SOperatorInfo* pOperator) {
int32_t numOfCols = 0;
int32_t vgId = 0;
int64_t dbSize = 0;
int64_t memSize = 0;
int64_t l1Size = 0;
int64_t l2Size = 0;
int64_t l3Size = 0;
int64_t walSize = 0;
int64_t metaSize = 0;
int64_t s3Size = 0;
pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL);
code = pAPI->metaFn.getDBSize(pInfo->readHandle.vnode, &dbSize, &walSize, &metaSize);
SDbSizeStatisInfo info = {0};
info.vgId = vgId;
code = pAPI->metaFn.getDBSize(pInfo->readHandle.vnode, &info);
QUERY_CHECK_CODE(code, lino, _end);
SName sn = {0};
@ -2072,8 +2080,6 @@ static SSDataBlock* sysTableBuildVgUsage(SOperatorInfo* pOperator) {
code = blockDataEnsureCapacity(p, pOperator->resultInfo.capacity);
QUERY_CHECK_CODE(code, lino, _end);
char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, numOfCols++);
code = colDataSetVal(pColInfoData, numOfRows, dbname, false);
QUERY_CHECK_CODE(code, lino, _end);
@ -2089,30 +2095,30 @@ static SSDataBlock* sysTableBuildVgUsage(SOperatorInfo* pOperator) {
totalSize += walSize;
pColInfoData = taosArrayGet(p->pDataBlock, numOfCols++);
code = colDataSetVal(pColInfoData, numOfRows, (char*)&walSize, false); // memtable
code = colDataSetVal(pColInfoData, numOfRows, (char*)&memSize, false); // memtable
QUERY_CHECK_CODE(code, lino, _end);
totalSize += walSize;
pColInfoData = taosArrayGet(p->pDataBlock, numOfCols++);
code = colDataSetVal(pColInfoData, numOfRows, (char*)&walSize, false); // l1_size
code = colDataSetVal(pColInfoData, numOfRows, (char*)&l1Size, false); // l1_size
QUERY_CHECK_CODE(code, lino, _end);
totalSize += walSize;
pColInfoData = taosArrayGet(p->pDataBlock, numOfCols++);
code = colDataSetVal(pColInfoData, numOfRows, (char*)&walSize, false); // l2_size
code = colDataSetVal(pColInfoData, numOfRows, (char*)&l2Size, false); // l2_size
QUERY_CHECK_CODE(code, lino, _end);
totalSize += walSize;
pColInfoData = taosArrayGet(p->pDataBlock, numOfCols++);
code = colDataSetVal(pColInfoData, numOfRows, (char*)&walSize, false); // l3_size
code = colDataSetVal(pColInfoData, numOfRows, (char*)&l3Size, false); // l3_size
QUERY_CHECK_CODE(code, lino, _end);
totalSize += walSize;
pColInfoData = taosArrayGet(p->pDataBlock, numOfCols++);
code = colDataSetVal(pColInfoData, numOfRows, (char*)&walSize, false); // s3_size
code = colDataSetVal(pColInfoData, numOfRows, (char*)&s3Size, false); // s3_size
QUERY_CHECK_CODE(code, lino, _end);
totalSize += walSize;

69
source/os/src/osDir.c
View File

@ -146,7 +146,7 @@ int32_t taosMulMkDir(const char *dirname) {
int32_t code = 0;
#ifdef WINDOWS
code = taosRealPath(dirname, temp, sizeof(temp));
if(code != 0) {
if (code != 0) {
return code;
}
if (temp[1] == ':') pos += 3;
@ -189,7 +189,7 @@ int32_t taosMulMkDir(const char *dirname) {
code = mkdir(temp, 0755);
#endif
if (code < 0 && errno != EEXIST) {
terrno = TAOS_SYSTEM_ERROR(errno);
terrno = TAOS_SYSTEM_ERROR(errno);
return code;
}
}
@ -211,7 +211,7 @@ int32_t taosMulModeMkDir(const char *dirname, int mode, bool checkAccess) {
int32_t code = 0;
#ifdef WINDOWS
code = taosRealPath(dirname, temp, sizeof(temp));
if(code != 0) {
if (code != 0) {
return code;
}
if (temp[1] == ':') pos += 3;
@ -220,7 +220,8 @@ int32_t taosMulModeMkDir(const char *dirname, int mode, bool checkAccess) {
#endif
if (taosDirExist(temp)) {
if (checkAccess && taosCheckAccessFile(temp, TD_FILE_ACCESS_EXIST_OK | TD_FILE_ACCESS_READ_OK | TD_FILE_ACCESS_WRITE_OK)) {
if (checkAccess &&
taosCheckAccessFile(temp, TD_FILE_ACCESS_EXIST_OK | TD_FILE_ACCESS_READ_OK | TD_FILE_ACCESS_WRITE_OK)) {
return 0;
}
@ -274,7 +275,8 @@ int32_t taosMulModeMkDir(const char *dirname, int mode, bool checkAccess) {
}
if (code < 0 && errno == EEXIST) {
if (checkAccess && taosCheckAccessFile(temp, TD_FILE_ACCESS_EXIST_OK | TD_FILE_ACCESS_READ_OK | TD_FILE_ACCESS_WRITE_OK)) {
if (checkAccess &&
taosCheckAccessFile(temp, TD_FILE_ACCESS_EXIST_OK | TD_FILE_ACCESS_READ_OK | TD_FILE_ACCESS_WRITE_OK)) {
return 0;
}
}
@ -305,7 +307,7 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) {
int32_t len = (int32_t)strlen(filename);
if (len > 3 && strcmp(filename + len - 3, ".gz") == 0) {
len -= 3;
}else{
} else {
continue;
}
@ -321,7 +323,7 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) {
int32_t days = (int32_t)(TABS(sec - fileSec) / 86400 + 1);
if (days > keepDays) {
TAOS_UNUSED(taosRemoveFile(filename));
uInfo("file:%s is removed, days:%d keepDays:%d, sed:%"PRId64, filename, days, keepDays, fileSec);
uInfo("file:%s is removed, days:%d keepDays:%d, sed:%" PRId64, filename, days, keepDays, fileSec);
} else {
// printf("file:%s won't be removed, days:%d keepDays:%d", filename, days, keepDays);
}
@ -334,7 +336,7 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) {
int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) {
wordexp_t full_path;
int32_t code = wordexp(dirname, &full_path, 0);
int32_t code = wordexp(dirname, &full_path, 0);
switch (code) {
case 0:
break;
@ -441,7 +443,7 @@ TdDirPtr taosOpenDir(const char *dirname) {
HANDLE hFind;
TdDirPtr pDir = taosMemoryMalloc(sizeof(TdDir));
if(pDir == NULL) {
if (pDir == NULL) {
return NULL;
}
@ -478,7 +480,7 @@ TdDirPtr taosOpenDir(const char *dirname) {
TdDirEntryPtr taosReadDir(TdDirPtr pDir) {
if (pDir == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
#ifdef WINDOWS
@ -526,13 +528,13 @@ char *taosGetDirEntryName(TdDirEntryPtr pDirEntry) {
}
int32_t taosCloseDir(TdDirPtr *ppDir) {
int32_t code = 0;
int32_t code = 0;
if (ppDir == NULL || *ppDir == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
#ifdef WINDOWS
if(!FindClose((*ppDir)->hFind)) {
if (!FindClose((*ppDir)->hFind)) {
terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError());
return terrno;
}
@ -567,3 +569,46 @@ void taosGetCwd(char *buf, int32_t len) {
tstrncpy(buf, "not implemented on windows", len);
#endif
}
int taosGetDirSize(const char *path, int64_t *size) {
int32_t code;
TdDirPtr pDir = taosOpenDir(path);
if (pDir == NULL) {
return code = terrno;
}
int32_t nBytes = 0;
char fullPath[512] = {0};
int64_t totalSize = 0;
TdDirEntryPtr de = NULL;
while ((de = taosReadDir(pDir)) != NULL) {
char *name = taosGetDirEntryName(de);
if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) {
continue;
}
nBytes = snprintf(fullPath, sizeof(fullPath), "%s%s%s", path, TD_DIRSEP, name);
if (nBytes <= 0 || nBytes >= sizeof(fullPath)) {
return TSDB_CODE_OUT_OF_RANGE;
}
if (taosIsDir(fullPath)) {
code = taosGetDirSize(fullPath, &totalSize);
if (code != 0) {
taosCloseDir(&pDir);
return code;
}
} else {
int64_t fileSize = 0;
code = taosStatFile(fullPath, &fileSize, NULL, NULL);
if (code != 0) {
taosCloseDir(&pDir);
return code;
}
totalSize += fileSize;
}
fullPath[0] = 0;
}
*size = totalSize;
TAOS_UNUSED(taosCloseDir(&pDir));
return 0;
}

4
source/os/src/osFile.c
View File

@ -420,7 +420,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) {
}
int64_t res = 0;
DWORD bytesRead;
DWORD bytesRead;
if (!ReadFile(pFile->hFile, buf, count, &bytesRead, NULL)) {
DWORD errCode = GetLastError();
terrno = TAOS_SYSTEM_WINAPI_ERROR(errCode);
@ -1565,4 +1565,4 @@ int taosSetAutoDelFile(char *path) {
}
return 0;
#endif
}
}