xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #26160 from taosdata/enh/TD-30554-3.0 

tfs: support disable create new file
This commit is contained in:
Hongze Cheng 2024-06-20 16:46:00 +08:00 committed by GitHub
parent 05ebfa70d4 3ed71361bb
commit 70851f6bed
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 134 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/util/tdef.h
View File

@ -549,6 +549,7 @@ typedef struct {
char dir[TSDB_FILENAME_LEN]; char dir[TSDB_FILENAME_LEN];
int32_t level; int32_t level;
int32_t primary; int32_t primary;
int8_t disable; // disable create new file
} SDiskCfg; } SDiskCfg;
typedef struct { typedef struct {

1
source/common/src/tglobal.c
View File

@ -326,6 +326,7 @@ int32_t taosSetTfsCfg(SConfig *pCfg) {
tstrncpy(tsDiskCfg[0].dir, pItem->str, TSDB_FILENAME_LEN); tstrncpy(tsDiskCfg[0].dir, pItem->str, TSDB_FILENAME_LEN);
tsDiskCfg[0].level = 0; tsDiskCfg[0].level = 0;
tsDiskCfg[0].primary = 1; tsDiskCfg[0].primary = 1;
tsDiskCfg[0].disable = 0;
tstrncpy(tsDataDir, pItem->str, PATH_MAX); tstrncpy(tsDataDir, pItem->str, PATH_MAX);
if (taosMulMkDir(tsDataDir) != 0) { if (taosMulMkDir(tsDataDir) != 0) {
uError("failed to create dataDir:%s", tsDataDir); uError("failed to create dataDir:%s", tsDataDir);

4
source/dnode/mgmt/node_mgmt/src/dmEnv.c
View File

@ -110,10 +110,8 @@ static bool dmCheckDiskSpace() {
int32_t dmDiskInit() { int32_t dmDiskInit() {
SDnode *pDnode = dmInstance(); SDnode *pDnode = dmInstance();
SDiskCfg dCfg = {0}; SDiskCfg dCfg = {.level = 0, .primary = 1, .disable = 0};
tstrncpy(dCfg.dir, tsDataDir, TSDB_FILENAME_LEN); tstrncpy(dCfg.dir, tsDataDir, TSDB_FILENAME_LEN);
dCfg.level = 0;
dCfg.primary = 1;
SDiskCfg *pDisks = tsDiskCfg; SDiskCfg *pDisks = tsDiskCfg;
int32_t numOfDisks = tsDiskCfgNum; int32_t numOfDisks = tsDiskCfgNum;
if (numOfDisks <= 0 || pDisks == NULL) { if (numOfDisks <= 0 || pDisks == NULL) {

1
source/dnode/vnode/test/tsdbSmaTest.cpp
View File

@ -368,6 +368,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
SDiskCfg pDisks = {0}; SDiskCfg pDisks = {0};
pDisks.level = 0; pDisks.level = 0;
pDisks.primary = 1; pDisks.primary = 1;
pDisks.disable = 0;
strncpy(pDisks.dir, TD_DATA_DIR_PATH, TSDB_FILENAME_LEN); strncpy(pDisks.dir, TD_DATA_DIR_PATH, TSDB_FILENAME_LEN);
int32_t numOfDisks = 1; int32_t numOfDisks = 1;
pTsdb->pTfs = tfsOpen(&pDisks, numOfDisks); pTsdb->pTfs = tfsOpen(&pDisks, numOfDisks);

3
source/libs/tfs/inc/tfsInt.h
View File

@ -38,6 +38,7 @@
typedef struct { typedef struct {
int32_t level; int32_t level;
int32_t id; int32_t id;
int8_t disable; // disable create new file
char *path; char *path;
SDiskSize size; SDiskSize size;
} STfsDisk; } STfsDisk;
@ -73,7 +74,7 @@ typedef struct STfs {
SHashObj *hash; // name to did map SHashObj *hash; // name to did map
} STfs; } STfs;
STfsDisk *tfsNewDisk(int32_t level, int32_t id, const char *dir); STfsDisk *tfsNewDisk(int32_t level, int32_t id, int8_t disable, const char *dir);
STfsDisk *tfsFreeDisk(STfsDisk *pDisk); STfsDisk *tfsFreeDisk(STfsDisk *pDisk);
int32_t tfsUpdateDiskSize(STfsDisk *pDisk); int32_t tfsUpdateDiskSize(STfsDisk *pDisk);

21
source/libs/tfs/src/tfs.c
View File

@ -519,6 +519,18 @@ static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) {
return -1; return -1;
} }
if (pCfg->primary < 0 || pCfg->primary > 1) {
fError("failed to mount %s to FS since invalid primary %d", pCfg->dir, pCfg->primary);
terrno = TSDB_CODE_FS_INVLD_CFG;
return -1;
}
if (pCfg->disable < 0 || pCfg->disable > 1) {
fError("failed to mount %s to FS since invalid disable %" PRIi8, pCfg->dir, pCfg->disable);
terrno = TSDB_CODE_FS_INVLD_CFG;
return -1;
}
if (pCfg->primary) { if (pCfg->primary) {
if (pCfg->level != 0) { if (pCfg->level != 0) {
fError("failed to mount %s to FS since disk is primary but level %d not 0", pCfg->dir, pCfg->level); fError("failed to mount %s to FS since disk is primary but level %d not 0", pCfg->dir, pCfg->level);
@ -595,6 +607,15 @@ static int32_t tfsCheck(STfs *pTfs) {
terrno = TSDB_CODE_FS_NO_MOUNT_AT_TIER; terrno = TSDB_CODE_FS_NO_MOUNT_AT_TIER;
return -1; return -1;
} }
if (level == 0) {
tfsUpdateTierSize(TFS_TIER_AT(pTfs, level));
if (TFS_TIER_AT(pTfs, level)->nAvailDisks == 0) {
fError("no disk to create new file at level %d", level);
terrno = TSDB_CODE_FS_NO_VALID_DISK;
return -1;
}
}
} }
return 0; return 0;

3
source/libs/tfs/src/tfsDisk.c
View File

@ -16,7 +16,7 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "tfsInt.h" #include "tfsInt.h"
STfsDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) { STfsDisk *tfsNewDisk(int32_t level, int32_t id, int8_t disable, const char *path) {
STfsDisk *pDisk = taosMemoryCalloc(1, sizeof(STfsDisk)); STfsDisk *pDisk = taosMemoryCalloc(1, sizeof(STfsDisk));
if (pDisk == NULL) { if (pDisk == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -32,6 +32,7 @@ STfsDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) {
pDisk->level = level; pDisk->level = level;
pDisk->id = id; pDisk->id = id;
pDisk->disable = disable;
taosGetDiskSize(pDisk->path, &pDisk->size); taosGetDiskSize(pDisk->path, &pDisk->size);
return pDisk; return pDisk;
} }

10
source/libs/tfs/src/tfsTier.c
View File

@ -65,7 +65,7 @@ STfsDisk *tfsMountDiskToTier(STfsTier *pTier, SDiskCfg *pCfg) {
return NULL; return NULL;
} }
STfsDisk *pDisk = tfsNewDisk(pCfg->level, id, pCfg->dir); STfsDisk *pDisk = tfsNewDisk(pCfg->level, id, pCfg->disable, pCfg->dir);
if (pDisk == NULL) return NULL; if (pDisk == NULL) return NULL;
pTier->disks[id] = pDisk; pTier->disks[id] = pDisk;
@ -89,7 +89,7 @@ void tfsUpdateTierSize(STfsTier *pTier) {
size.total += pDisk->size.total; size.total += pDisk->size.total;
size.used += pDisk->size.used; size.used += pDisk->size.used;
size.avail += pDisk->size.avail; size.avail += pDisk->size.avail;
nAvailDisks++; if (pDisk->disable == 0) nAvailDisks++;
} }
pTier->size = size; pTier->size = size;
@ -118,6 +118,12 @@ int32_t tfsAllocDiskOnTier(STfsTier *pTier) {
if (pDisk == NULL) continue; if (pDisk == NULL) continue;
if (pDisk->disable == 1) {
uTrace("disk %s is disabled and skip it, level:%d id:%d disable:%" PRIi8, pDisk->path, pDisk->level, pDisk->id,
pDisk->disable);
continue;
}
if (pDisk->size.avail < tsMinDiskFreeSize) { if (pDisk->size.avail < tsMinDiskFreeSize) {
uInfo("disk %s is full and skip it, level:%d id:%d free size:%" PRId64 " min free size:%" PRId64, pDisk->path, uInfo("disk %s is full and skip it, level:%d id:%d free size:%" PRId64 " min free size:%" PRId64, pDisk->path,
pDisk->level, pDisk->id, pDisk->size.avail, tsMinDiskFreeSize); pDisk->level, pDisk->id, pDisk->size.avail, tsMinDiskFreeSize);

12
source/libs/tfs/test/tfsTest.cpp
View File

@ -37,6 +37,7 @@ TEST_F(TfsTest, 01_Open_Close) {
tstrncpy(dCfg.dir, root, TSDB_FILENAME_LEN); tstrncpy(dCfg.dir, root, TSDB_FILENAME_LEN);
dCfg.level = 0; dCfg.level = 0;
dCfg.primary = 1; dCfg.primary = 1;
dCfg.disable = 0;
taosRemoveDir(root); taosRemoveDir(root);
STfs *pTfs = tfsOpen(&dCfg, 1); STfs *pTfs = tfsOpen(&dCfg, 1);
@ -63,6 +64,7 @@ TEST_F(TfsTest, 02_AllocDisk) {
tstrncpy(dCfg.dir, root, TSDB_FILENAME_LEN); tstrncpy(dCfg.dir, root, TSDB_FILENAME_LEN);
dCfg.level = 0; dCfg.level = 0;
dCfg.primary = 1; dCfg.primary = 1;
dCfg.disable = 0;
taosRemoveDir(root); taosRemoveDir(root);
taosMkDir(root); taosMkDir(root);
@ -114,6 +116,7 @@ TEST_F(TfsTest, 03_Dir) {
tstrncpy(dCfg.dir, root, TSDB_FILENAME_LEN); tstrncpy(dCfg.dir, root, TSDB_FILENAME_LEN);
dCfg.level = 0; dCfg.level = 0;
dCfg.primary = 1; dCfg.primary = 1;
dCfg.disable = 0;
taosRemoveDir(root); taosRemoveDir(root);
taosMkDir(root); taosMkDir(root);
@ -330,30 +333,39 @@ TEST_F(TfsTest, 05_MultiDisk) {
tstrncpy(dCfg[0].dir, root01, TSDB_FILENAME_LEN); tstrncpy(dCfg[0].dir, root01, TSDB_FILENAME_LEN);
dCfg[0].level = 0; dCfg[0].level = 0;
dCfg[0].primary = 0; dCfg[0].primary = 0;
dCfg[0].disable = 0;
tstrncpy(dCfg[1].dir, root00, TSDB_FILENAME_LEN); tstrncpy(dCfg[1].dir, root00, TSDB_FILENAME_LEN);
dCfg[1].level = 0; dCfg[1].level = 0;
dCfg[1].primary = 0; dCfg[1].primary = 0;
dCfg[1].disable = 0;
tstrncpy(dCfg[2].dir, root20, TSDB_FILENAME_LEN); tstrncpy(dCfg[2].dir, root20, TSDB_FILENAME_LEN);
dCfg[2].level = 2; dCfg[2].level = 2;
dCfg[2].primary = 0; dCfg[2].primary = 0;
dCfg[2].disable = 0;
tstrncpy(dCfg[3].dir, root21, TSDB_FILENAME_LEN); tstrncpy(dCfg[3].dir, root21, TSDB_FILENAME_LEN);
dCfg[3].level = 2; dCfg[3].level = 2;
dCfg[3].primary = 0; dCfg[3].primary = 0;
dCfg[3].disable = 0;
tstrncpy(dCfg[4].dir, root22, TSDB_FILENAME_LEN); tstrncpy(dCfg[4].dir, root22, TSDB_FILENAME_LEN);
dCfg[4].level = 2; dCfg[4].level = 2;
dCfg[4].primary = 0; dCfg[4].primary = 0;
dCfg[4].disable = 0;
tstrncpy(dCfg[5].dir, root23, TSDB_FILENAME_LEN); tstrncpy(dCfg[5].dir, root23, TSDB_FILENAME_LEN);
dCfg[5].level = 2; dCfg[5].level = 2;
dCfg[5].primary = 0; dCfg[5].primary = 0;
dCfg[5].disable = 0;
tstrncpy(dCfg[6].dir, root10, TSDB_FILENAME_LEN); tstrncpy(dCfg[6].dir, root10, TSDB_FILENAME_LEN);
dCfg[6].level = 1; dCfg[6].level = 1;
dCfg[6].primary = 0; dCfg[6].primary = 0;
dCfg[6].disable = 0;
tstrncpy(dCfg[7].dir, root11, TSDB_FILENAME_LEN); tstrncpy(dCfg[7].dir, root11, TSDB_FILENAME_LEN);
dCfg[7].level = 1; dCfg[7].level = 1;
dCfg[7].primary = 0; dCfg[7].primary = 0;
dCfg[7].disable = 0;
tstrncpy(dCfg[8].dir, root12, TSDB_FILENAME_LEN); tstrncpy(dCfg[8].dir, root12, TSDB_FILENAME_LEN);
dCfg[8].level = 1; dCfg[8].level = 1;
dCfg[8].primary = 0; dCfg[8].primary = 0;
dCfg[8].disable = 0;
taosRemoveDir(root00); taosRemoveDir(root00);
taosRemoveDir(root01); taosRemoveDir(root01);

79
source/util/src/tconfig.c
View File

@ -259,7 +259,7 @@ static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType
} }
static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value, const char *level, const char *primary, static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value, const char *level, const char *primary,
ECfgSrcType stype) { const char *disable, ECfgSrcType stype) {
taosThreadMutexLock(&pCfg->lock); taosThreadMutexLock(&pCfg->lock);
SConfigItem *pItem = cfgGetItem(pCfg, name); SConfigItem *pItem = cfgGetItem(pCfg, name);
@ -283,6 +283,7 @@ static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value,
tstrncpy(cfg.dir, pItem->str, sizeof(cfg.dir)); tstrncpy(cfg.dir, pItem->str, sizeof(cfg.dir));
cfg.level = level ? atoi(level) : 0; cfg.level = level ? atoi(level) : 0;
cfg.primary = primary ? atoi(primary) : 1; cfg.primary = primary ? atoi(primary) : 1;
cfg.disable = disable ? atoi(disable) : 0;
void *ret = taosArrayPush(pItem->array, &cfg); void *ret = taosArrayPush(pItem->array, &cfg);
if (ret == NULL) { if (ret == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -899,16 +900,16 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
} }
int32_t cfgLoadFromEnvVar(SConfig *pConfig) { int32_t cfgLoadFromEnvVar(SConfig *pConfig) {
char line[1024], *name, *value, *value2, *value3; char line[1024], *name, *value, *value2, *value3, *value4;
int32_t olen, vlen, vlen2, vlen3; int32_t olen, vlen, vlen2, vlen3, vlen4;
int32_t code = 0; int32_t code = 0;
char **pEnv = environ; char **pEnv = environ;
line[1023] = 0; line[1023] = 0;
if (pEnv == NULL) return 0; if (pEnv == NULL) return 0;
while (*pEnv != NULL) { while (*pEnv != NULL) {
name = value = value2 = value3 = NULL; name = value = value2 = value3 = value4 = NULL;
olen = vlen = vlen2 = vlen3 = 0; olen = vlen = vlen2 = vlen3 = vlen4 = 0;
strncpy(line, *pEnv, sizeof(line) - 1); strncpy(line, *pEnv, sizeof(line) - 1);
pEnv++; pEnv++;
@ -926,14 +927,18 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) {
if (vlen2 != 0) { if (vlen2 != 0) {
value2[vlen2] = 0; value2[vlen2] = 0;
paGetToken(value2 + vlen2 + 1, &value3, &vlen3); paGetToken(value2 + vlen2 + 1, &value3, &vlen3);
if (vlen3 != 0) value3[vlen3] = 0; if (vlen3 != 0) {
value3[vlen3] = 0;
paGetToken(value3 + vlen3 + 1, &value4, &vlen4);
if(vlen4 != 0) value4[vlen4] = 0;
}
} }
code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_VAR, true); code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_VAR, true);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
if (strcasecmp(name, "dataDir") == 0) { if (strcasecmp(name, "dataDir") == 0) {
code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_VAR); code = cfgSetTfsItem(pConfig, name, value, value2, value3, value4, CFG_STYPE_ENV_VAR);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
} }
} }
@ -943,8 +948,8 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) {
} }
int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) { int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) {
char buf[1024], *name, *value, *value2, *value3; char buf[1024], *name, *value, *value2, *value3, *value4;
int32_t olen, vlen, vlen2, vlen3; int32_t olen, vlen, vlen2, vlen3, vlen4;
int32_t code = 0; int32_t code = 0;
int32_t index = 0; int32_t index = 0;
if (envCmd == NULL) return 0; if (envCmd == NULL) return 0;
@ -954,8 +959,8 @@ int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) {
taosEnvToCfg(buf, buf); taosEnvToCfg(buf, buf);
index++; index++;
name = value = value2 = value3 = NULL; name = value = value2 = value3 = value4 = NULL;
olen = vlen = vlen2 = vlen3 = 0; olen = vlen = vlen2 = vlen3 = vlen4 = 0;
paGetToken(buf, &name, &olen); paGetToken(buf, &name, &olen);
if (olen == 0) continue; if (olen == 0) continue;
@ -969,14 +974,18 @@ int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) {
if (vlen2 != 0) { if (vlen2 != 0) {
value2[vlen2] = 0; value2[vlen2] = 0;
paGetToken(value2 + vlen2 + 1, &value3, &vlen3); paGetToken(value2 + vlen2 + 1, &value3, &vlen3);
if (vlen3 != 0) value3[vlen3] = 0; if (vlen3 != 0) {
value3[vlen3] = 0;
paGetToken(value3 + vlen3 + 1, &value4, &vlen4);
if(vlen4 != 0) value4[vlen4] = 0;
}
} }
code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_CMD, true); code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_CMD, true);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
if (strcasecmp(name, "dataDir") == 0) { if (strcasecmp(name, "dataDir") == 0) {
code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_CMD); code = cfgSetTfsItem(pConfig, name, value, value2, value3, value4, CFG_STYPE_ENV_CMD);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
} }
} }
@ -986,8 +995,8 @@ int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) {
} }
int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) { int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) {
char line[1024], *name, *value, *value2, *value3; char line[1024], *name, *value, *value2, *value3, *value4;
int32_t olen, vlen, vlen2, vlen3; int32_t olen, vlen, vlen2, vlen3, vlen4;
int32_t code = 0; int32_t code = 0;
ssize_t _bytes = 0; ssize_t _bytes = 0;
@ -1012,8 +1021,8 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) {
} }
while (!taosEOFFile(pFile)) { while (!taosEOFFile(pFile)) {
name = value = value2 = value3 = NULL; name = value = value2 = value3 = value4 = NULL;
olen = vlen = vlen2 = vlen3 = 0; olen = vlen = vlen2 = vlen3 = vlen4 = 0;
_bytes = taosGetsFile(pFile, sizeof(line), line); _bytes = taosGetsFile(pFile, sizeof(line), line);
if (_bytes <= 0) { if (_bytes <= 0) {
@ -1034,14 +1043,18 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) {
if (vlen2 != 0) { if (vlen2 != 0) {
value2[vlen2] = 0; value2[vlen2] = 0;
paGetToken(value2 + vlen2 + 1, &value3, &vlen3); paGetToken(value2 + vlen2 + 1, &value3, &vlen3);
if (vlen3 != 0) value3[vlen3] = 0; if (vlen3 != 0) {
value3[vlen3] = 0;
paGetToken(value3 + vlen3 + 1, &value4, &vlen4);
if(vlen4 != 0) value4[vlen4] = 0;
}
} }
code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_FILE, true); code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_FILE, true);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
if (strcasecmp(name, "dataDir") == 0) { if (strcasecmp(name, "dataDir") == 0) {
code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_FILE); code = cfgSetTfsItem(pConfig, name, value, value2, value3, value4, CFG_STYPE_ENV_FILE);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
} }
} }
@ -1053,8 +1066,8 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) {
} }
int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
char line[1024], *name, *value, *value2, *value3; char line[1024], *name, *value, *value2, *value3, *value4;
int32_t olen, vlen, vlen2, vlen3; int32_t olen, vlen, vlen2, vlen3, vlen4;
ssize_t _bytes = 0; ssize_t _bytes = 0;
int32_t code = 0; int32_t code = 0;
@ -1072,8 +1085,8 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
} }
while (!taosEOFFile(pFile)) { while (!taosEOFFile(pFile)) {
name = value = value2 = value3 = NULL; name = value = value2 = value3 = value4 = NULL;
olen = vlen = vlen2 = vlen3 = 0; olen = vlen = vlen2 = vlen3 = vlen4 = 0;
_bytes = taosGetsFile(pFile, sizeof(line), line); _bytes = taosGetsFile(pFile, sizeof(line), line);
if (_bytes <= 0) { if (_bytes <= 0) {
@ -1114,7 +1127,11 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
if (vlen2 != 0) { if (vlen2 != 0) {
value2[vlen2] = 0; value2[vlen2] = 0;
paGetToken(value2 + vlen2 + 1, &value3, &vlen3); paGetToken(value2 + vlen2 + 1, &value3, &vlen3);
if (vlen3 != 0) value3[vlen3] = 0; if (vlen3 != 0) {
value3[vlen3] = 0;
paGetToken(value3 + vlen3 + 1, &value4, &vlen4);
if (vlen4 != 0) value4[vlen4] = 0;
}
} }
code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE, true); code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE, true);
@ -1122,7 +1139,7 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
} }
if (strcasecmp(name, "dataDir") == 0) { if (strcasecmp(name, "dataDir") == 0) {
code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE); code = cfgSetTfsItem(pConfig, name, value, value2, value3, value4, CFG_STYPE_CFG_FILE);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
} }
@ -1212,8 +1229,8 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
// } // }
int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) {
char *cfgLineBuf = NULL, *name, *value, *value2, *value3; char *cfgLineBuf = NULL, *name, *value, *value2, *value3, *value4;
int32_t olen, vlen, vlen2, vlen3; int32_t olen, vlen, vlen2, vlen3, vlen4;
int32_t code = 0; int32_t code = 0;
if (url == NULL || strlen(url) == 0) { if (url == NULL || strlen(url) == 0) {
uInfo("apoll url not load"); uInfo("apoll url not load");
@ -1289,14 +1306,18 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) {
if (vlen2 != 0) { if (vlen2 != 0) {
value2[vlen2] = 0; value2[vlen2] = 0;
paGetToken(value2 + vlen2 + 1, &value3, &vlen3); paGetToken(value2 + vlen2 + 1, &value3, &vlen3);
if (vlen3 != 0) value3[vlen3] = 0; if (vlen3 != 0) {
value3[vlen3] = 0;
paGetToken(value3 + vlen3 + 1, &value4, &vlen4);
if (vlen4 != 0) value4[vlen4] = 0;
}
} }
code = cfgSetItem(pConfig, name, value, CFG_STYPE_APOLLO_URL, true); code = cfgSetItem(pConfig, name, value, CFG_STYPE_APOLLO_URL, true);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
if (strcasecmp(name, "dataDir") == 0) { if (strcasecmp(name, "dataDir") == 0) {
code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_APOLLO_URL); code = cfgSetTfsItem(pConfig, name, value, value2, value3, value4, CFG_STYPE_APOLLO_URL);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
} }
} }

48
tests/system-test/0-others/multilevel.py
View File

@ -52,7 +52,7 @@ class TDTestCase:
tdLog.info("============== basic test ===============") tdLog.info("============== basic test ===============")
cfg={ cfg={
'/mnt/data1' : 'dataDir', '/mnt/data1' : 'dataDir',
'/mnt/data2 0 0' : 'dataDir' '/mnt/data2 0 0 0' : 'dataDir'
} }
tdSql.createDir('/mnt/data1') tdSql.createDir('/mnt/data1')
tdSql.createDir('/mnt/data2') tdSql.createDir('/mnt/data2')
@ -112,9 +112,9 @@ class TDTestCase:
cfg={ cfg={
'/mnt/data00 0 1' : 'dataDir', '/mnt/data00 0 1' : 'dataDir',
'/mnt/data01 0 0' : 'dataDir', '/mnt/data01 0 0' : 'dataDir',
'/mnt/data02 0 0' : 'dataDir', '/mnt/data02 0 0 0' : 'dataDir',
'/mnt/data03 0 0' : 'dataDir', '/mnt/data03 0 0' : 'dataDir',
'/mnt/data04 0 0' : 'dataDir' '/mnt/data04 0 0 0' : 'dataDir'
} }
dir_list = ['/mnt/data00','/mnt/data01','/mnt/data02','/mnt/data03','/mnt/data04'] dir_list = ['/mnt/data00','/mnt/data01','/mnt/data02','/mnt/data03','/mnt/data04']
for i in dir_list: for i in dir_list:
@ -160,15 +160,15 @@ class TDTestCase:
tdDnodes.stop(1) tdDnodes.stop(1)
# Test1 1 dataDir # Test1 1 dataDir
cfg={ cfg={
'/mnt/data000 0 1' : 'dataDir', '/mnt/data000 0 1 0' : 'dataDir',
'/mnt/data001 0 0' : 'dataDir', '/mnt/data001 0 0' : 'dataDir',
'/mnt/data002 0 0' : 'dataDir', '/mnt/data002 0 0 0' : 'dataDir',
'/mnt/data010 1 0' : 'dataDir', '/mnt/data010 1 0' : 'dataDir',
'/mnt/data011 1 0' : 'dataDir', '/mnt/data011 1 0 0' : 'dataDir',
'/mnt/data012 1 0' : 'dataDir', '/mnt/data012 1 0' : 'dataDir',
'/mnt/data020 2 0' : 'dataDir', '/mnt/data020 2 0 0' : 'dataDir',
'/mnt/data021 2 0' : 'dataDir', '/mnt/data021 2 0 0' : 'dataDir',
'/mnt/data022 2 0' : 'dataDir' '/mnt/data022 2 0 0' : 'dataDir'
} }
dir_list = ['/mnt/data000','/mnt/data001','/mnt/data002','/mnt/data010','/mnt/data011','/mnt/data012','/mnt/data020','/mnt/data021''/mnt/data022'] dir_list = ['/mnt/data000','/mnt/data001','/mnt/data002','/mnt/data010','/mnt/data011','/mnt/data012','/mnt/data020','/mnt/data021''/mnt/data022']
for i in dir_list: for i in dir_list:
@ -189,7 +189,7 @@ class TDTestCase:
if i == 0 : if i == 0 :
datadir = '/mnt/data%d 0 1' % (i+1) datadir = '/mnt/data%d 0 1' % (i+1)
else: else:
datadir = '/mnt/data%d 0 0' % (i+1) datadir = '/mnt/data%d 0 0 0' % (i+1)
cfg.update({ datadir : 'dataDir' }) cfg.update({ datadir : 'dataDir' })
tdSql.createDir('/mnt/data%d' % (i+1)) tdSql.createDir('/mnt/data%d' % (i+1))
@ -227,6 +227,22 @@ class TDTestCase:
tdSql.taosdStatus(0) tdSql.taosdStatus(0)
def disable_create_new_file(self):
tdLog.info("============== disable_create_new_file_0_not_exist test ===============")
cfg={
'/mnt/data1 0 1 1' : 'dataDir',
'/mnt/data2 0 0 1' : 'dataDir'
}
tdSql.createDir('/mnt/data1')
tdSql.createDir('/mnt/data2')
tdLog.info("================= step1")
tdDnodes.deploy(1,cfg)
tdDnodes.startWithoutSleep(1)
tdLog.info("================= step2")
tdSql.taosdStatus(0)
def trim_database(self): def trim_database(self):
tdLog.info("============== trim_database test ===============") tdLog.info("============== trim_database test ===============")
tdDnodes.stop(1) tdDnodes.stop(1)
@ -242,7 +258,7 @@ class TDTestCase:
tdSql.execute('use dbtest') tdSql.execute('use dbtest')
tdSql.execute('create table stb (ts timestamp,c0 int) tags(t0 int)') tdSql.execute('create table stb (ts timestamp,c0 int) tags(t0 int)')
tdSql.execute('create table tb1 using stb tags(1)') tdSql.execute('create table tb1 using stb tags(1)')
for i in range(10,30): for i in range(1,600, 30):
tdSql.execute(f'insert into tb1 values(now-{i}d,10)') tdSql.execute(f'insert into tb1 values(now-{i}d,10)')
tdSql.execute('flush database dbtest') tdSql.execute('flush database dbtest')
time.sleep(3) time.sleep(3)
@ -250,21 +266,26 @@ class TDTestCase:
tdDnodes.stop(1) tdDnodes.stop(1)
cfg={ cfg={
'/mnt/data1 0 1' : 'dataDir', '/mnt/data1 0 1' : 'dataDir',
'/mnt/data2 1 0' : 'dataDir', '/mnt/data2 1 0 0' : 'dataDir',
'/mnt/data3 2 0' : 'dataDir', '/mnt/data3 1 0 1' : 'dataDir',
'/mnt/data4 2 0' : 'dataDir',
} }
tdSql.createDir('/mnt/data2') tdSql.createDir('/mnt/data2')
tdSql.createDir('/mnt/data3') tdSql.createDir('/mnt/data3')
tdSql.createDir('/mnt/data4')
tdDnodes.deploy(1,cfg) tdDnodes.deploy(1,cfg)
tdDnodes.start(1) tdDnodes.start(1)
checkFiles('/mnt/data1/vnode/*/tsdb/v*',1) checkFiles('/mnt/data1/vnode/*/tsdb/v*',1)
checkFiles('/mnt/data2/vnode/*/tsdb/v*',0) checkFiles('/mnt/data2/vnode/*/tsdb/v*',0)
checkFiles('/mnt/data3/vnode/*/tsdb/v*',0) checkFiles('/mnt/data3/vnode/*/tsdb/v*',0)
checkFiles('/mnt/data4/vnode/*/tsdb/v*',0)
tdSql.execute('alter database dbtest keep 10d,365d,3650d') tdSql.execute('alter database dbtest keep 10d,365d,3650d')
tdSql.execute('trim database dbtest') tdSql.execute('trim database dbtest')
time.sleep(3) time.sleep(3)
checkFiles('/mnt/data1/vnode/*/tsdb/v*',1) checkFiles('/mnt/data1/vnode/*/tsdb/v*',1)
checkFiles('/mnt/data2/vnode/*/tsdb/v*',1) checkFiles('/mnt/data2/vnode/*/tsdb/v*',1)
checkFiles('/mnt/data3/vnode/*/tsdb/v*',0)
checkFiles('/mnt/data4/vnode/*/tsdb/v*',1)
def run(self): def run(self):
self.basic() self.basic()
@ -275,6 +296,7 @@ class TDTestCase:
self.more_than_128_disks() self.more_than_128_disks()
self.trim_database() self.trim_database()
self.missing_middle_level() self.missing_middle_level()
self.disable_create_new_file()