From e69fbd54a2b242da92905810aaf330a8abaac261 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 7 Sep 2020 14:35:56 +0000 Subject: [PATCH 1/6] [TD-1263] add config items, logKeepDays and logbakDir --- src/client/src/tscSystem.c | 4 ++++ src/common/inc/tglobal.h | 2 ++ src/common/src/tglobal.c | 20 ++++++++++++++++++++ src/dnode/src/dnodeMain.c | 5 +++++ src/os/src/alpine/alpineEnv.c | 1 + src/os/src/darwin/darwinEnv.c | 1 + src/os/src/linux/linuxEnv.c | 2 ++ src/os/src/windows/wEnv.c | 2 ++ src/util/src/tconfig.c | 6 +++--- src/util/src/tlog.c | 3 +++ 10 files changed, 43 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 72f23881d2..15f06f5a3b 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -94,6 +94,10 @@ void taos_init_imp(void) { printf("failed to create log dir:%s\n", tsLogDir); } + if (mkdir(tsLogbakDir, 0755) != 0 && errno != EEXIST) { + printf("failed to create logbak dir:%s\n", tsLogbakDir); + } + sprintf(temp, "%s/taoslog", tsLogDir); if (taosInitLog(temp, tsNumOfLogLines, 10) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index fedafe5b02..bf9859d58c 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -126,6 +126,7 @@ extern char tsDnodeDir[]; extern char tsMnodeDir[]; extern char tsDataDir[]; extern char tsLogDir[]; +extern char tsLogbakDir[]; extern char tsScriptDir[]; extern int64_t tsMsPerDay[3]; extern char tsVnodeBakDir[]; @@ -158,6 +159,7 @@ extern char buildinfo[]; // log extern int32_t tsAsyncLog; extern int32_t tsNumOfLogLines; +extern int32_t tsLogKeepDays; extern int32_t dDebugFlag; extern int32_t vDebugFlag; extern int32_t mDebugFlag; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 9683a63503..acbfd34d10 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -370,6 +370,16 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + cfg.option = "logBakDir"; + cfg.ptr = tsLogbakDir; + cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_LOG; + cfg.minValue = 0; + cfg.maxValue = 0; + cfg.ptrLength = TSDB_FILENAME_LEN; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + cfg.option = "scriptDir"; cfg.ptr = tsScriptDir; cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; @@ -1019,6 +1029,16 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + cfg.option = "logKeepDays"; + cfg.ptr = &tsNumOfLogLines; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; + cfg.minValue = 0; + cfg.maxValue = 3650; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + cfg.option = "asyncLog"; cfg.ptr = &tsAsyncLog; cfg.valType = TAOS_CFG_VTYPE_INT16; diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 97e6f2ce6d..a00124098a 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -106,6 +106,11 @@ int32_t dnodeInitSystem() { return -1; } + if (dnodeCreateDir(tsLogbakDir) < 0) { + printf("failed to create dir: %s, reason: %s\n", tsLogbakDir, strerror(errno)); + return -1; + } + char temp[TSDB_FILENAME_LEN]; sprintf(temp, "%s/taosdlog", tsLogDir); if (taosInitLog(temp, tsNumOfLogLines, 1) < 0) { diff --git a/src/os/src/alpine/alpineEnv.c b/src/os/src/alpine/alpineEnv.c index 811d98ad7f..972a7ae7f1 100644 --- a/src/os/src/alpine/alpineEnv.c +++ b/src/os/src/alpine/alpineEnv.c @@ -27,6 +27,7 @@ void osInit() { strcpy(tsMnodeDir, ""); strcpy(tsDataDir, "/var/lib/taos"); strcpy(tsLogDir, "/var/log/taos"); + strcpy(tsLogbakDir, "/var/log/taos/bak"); strcpy(tsScriptDir, "/etc/taos"); strcpy(tsOsName, "Linux"); } \ No newline at end of file diff --git a/src/os/src/darwin/darwinEnv.c b/src/os/src/darwin/darwinEnv.c index 6adebabec0..79f01c2459 100644 --- a/src/os/src/darwin/darwinEnv.c +++ b/src/os/src/darwin/darwinEnv.c @@ -27,6 +27,7 @@ void osInit() { strcpy(tsMnodeDir, ""); strcpy(tsDataDir, "~/TDengine/data"); strcpy(tsLogDir, "~/TDengine/log"); + strcpy(tsLogbakDir, "~/TDengine/log/bak"); strcpy(tsScriptDir, "~/TDengine/cfg"); strcpy(tsOsName, "Darwin"); } diff --git a/src/os/src/linux/linuxEnv.c b/src/os/src/linux/linuxEnv.c index 14b40a1f18..af3af97df2 100644 --- a/src/os/src/linux/linuxEnv.c +++ b/src/os/src/linux/linuxEnv.c @@ -25,6 +25,7 @@ void osInit() { } strcpy(tsDataDir, "/var/lib/power"); strcpy(tsLogDir, "/var/log/power"); + strcpy(tsLogbakDir, "/var/log/power/bak"); strcpy(tsScriptDir, "/etc/power"); #else if (configDir[0] == 0) { @@ -32,6 +33,7 @@ void osInit() { } strcpy(tsDataDir, "/var/lib/taos"); strcpy(tsLogDir, "/var/log/taos"); + strcpy(tsLogbakDir, "/var/log/taos/bak"); strcpy(tsScriptDir, "/etc/taos"); #endif diff --git a/src/os/src/windows/wEnv.c b/src/os/src/windows/wEnv.c index 8110a19490..2a758f31c3 100644 --- a/src/os/src/windows/wEnv.c +++ b/src/os/src/windows/wEnv.c @@ -30,6 +30,7 @@ void osInit() { strcpy(tsVnodeDir, "C:/PowerDB/data"); strcpy(tsDataDir, "C:/PowerDB/data"); strcpy(tsLogDir, "C:/PowerDB/log"); + strcpy(tsLogbakDir, "C:/PowerDB/log/bak"); strcpy(tsScriptDir, "C:/PowerDB/script"); #else @@ -40,6 +41,7 @@ void osInit() { strcpy(tsVnodeDir, "C:/TDengine/data"); strcpy(tsDataDir, "C:/TDengine/data"); strcpy(tsLogDir, "C:/TDengine/log"); + strcpy(tsLogbakDir, "C:/TDengine/log/bak"); strcpy(tsScriptDir, "C:/TDengine/script"); #endif diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index 875c597008..0ec55841a0 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -270,7 +270,7 @@ void taosReadGlobalLogCfg() { } wordfree(&full_path); - taosReadLogOption("tsLogDir", tsLogDir); + taosReadLogOption("logDir", tsLogDir); sprintf(fileName, "%s/taos.cfg", configDir); fp = fopen(fileName, "r"); @@ -288,9 +288,9 @@ void taosReadGlobalLogCfg() { option = value = NULL; olen = vlen = 0; - taosGetline(&line, &len, fp); + taosGetline(&line, &len, fp); line[len - 1] = 0; - + paGetToken(line, &option, &olen); if (olen == 0) continue; option[olen] = 0; diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index 766301914a..ab129b0ad4 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -62,14 +62,17 @@ typedef struct { pthread_mutex_t logMutex; } SLogObj; +int32_t tsLogKeepDays = 0; int32_t tsAsyncLog = 1; float tsTotalLogDirGB = 0; float tsAvailLogDirGB = 0; float tsMinimalLogDirGB = 0.1f; #ifdef _TD_POWER_ char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/power"; +char tsLogbakDir[TSDB_FILENAME_LEN] = "/var/log/power/bak"; #else char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/taos"; +char tsLogbakDir[TSDB_FILENAME_LEN] = "/var/log/taos/bak"; #endif static SLogObj tsLogObj = { .fileNum = 1 }; From 37922f299f3f26be48cf550092a0153012e285ec Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 8 Sep 2020 04:27:55 +0000 Subject: [PATCH 2/6] TD-1263 --- src/client/src/tscSystem.c | 4 --- src/common/inc/tglobal.h | 1 - src/common/src/tglobal.c | 10 ------- src/dnode/src/dnodeMain.c | 5 ---- src/os/inc/osDir.h | 1 + src/os/src/alpine/alpineEnv.c | 1 - src/os/src/darwin/darwinEnv.c | 1 - src/os/src/detail/osDir.c | 49 +++++++++++++++++++++++++++++++---- src/os/src/linux/linuxEnv.c | 2 -- src/os/src/windows/wEnv.c | 2 -- src/util/src/tlog.c | 23 +++++++++++++--- src/vnode/src/vnodeMain.c | 8 +++++- 12 files changed, 71 insertions(+), 36 deletions(-) diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 15f06f5a3b..72f23881d2 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -94,10 +94,6 @@ void taos_init_imp(void) { printf("failed to create log dir:%s\n", tsLogDir); } - if (mkdir(tsLogbakDir, 0755) != 0 && errno != EEXIST) { - printf("failed to create logbak dir:%s\n", tsLogbakDir); - } - sprintf(temp, "%s/taoslog", tsLogDir); if (taosInitLog(temp, tsNumOfLogLines, 10) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index bf9859d58c..77e8b76456 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -126,7 +126,6 @@ extern char tsDnodeDir[]; extern char tsMnodeDir[]; extern char tsDataDir[]; extern char tsLogDir[]; -extern char tsLogbakDir[]; extern char tsScriptDir[]; extern int64_t tsMsPerDay[3]; extern char tsVnodeBakDir[]; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index acbfd34d10..5250d2947e 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -370,16 +370,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "logBakDir"; - cfg.ptr = tsLogbakDir; - cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FILENAME_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - cfg.option = "scriptDir"; cfg.ptr = tsScriptDir; cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index a00124098a..97e6f2ce6d 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -106,11 +106,6 @@ int32_t dnodeInitSystem() { return -1; } - if (dnodeCreateDir(tsLogbakDir) < 0) { - printf("failed to create dir: %s, reason: %s\n", tsLogbakDir, strerror(errno)); - return -1; - } - char temp[TSDB_FILENAME_LEN]; sprintf(temp, "%s/taosdlog", tsLogDir); if (taosInitLog(temp, tsNumOfLogLines, 1) < 0) { diff --git a/src/os/inc/osDir.h b/src/os/inc/osDir.h index e7dc04fd15..17683743e3 100644 --- a/src/os/inc/osDir.h +++ b/src/os/inc/osDir.h @@ -24,6 +24,7 @@ extern "C" { void taosRemoveDir(char *rootDir); int taosMkDir(const char *pathname, mode_t mode); void taosRename(char* oldName, char *newName); +void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays); #ifdef __cplusplus } diff --git a/src/os/src/alpine/alpineEnv.c b/src/os/src/alpine/alpineEnv.c index 972a7ae7f1..811d98ad7f 100644 --- a/src/os/src/alpine/alpineEnv.c +++ b/src/os/src/alpine/alpineEnv.c @@ -27,7 +27,6 @@ void osInit() { strcpy(tsMnodeDir, ""); strcpy(tsDataDir, "/var/lib/taos"); strcpy(tsLogDir, "/var/log/taos"); - strcpy(tsLogbakDir, "/var/log/taos/bak"); strcpy(tsScriptDir, "/etc/taos"); strcpy(tsOsName, "Linux"); } \ No newline at end of file diff --git a/src/os/src/darwin/darwinEnv.c b/src/os/src/darwin/darwinEnv.c index 79f01c2459..6adebabec0 100644 --- a/src/os/src/darwin/darwinEnv.c +++ b/src/os/src/darwin/darwinEnv.c @@ -27,7 +27,6 @@ void osInit() { strcpy(tsMnodeDir, ""); strcpy(tsDataDir, "~/TDengine/data"); strcpy(tsLogDir, "~/TDengine/log"); - strcpy(tsLogbakDir, "~/TDengine/log/bak"); strcpy(tsScriptDir, "~/TDengine/cfg"); strcpy(tsOsName, "Darwin"); } diff --git a/src/os/src/detail/osDir.c b/src/os/src/detail/osDir.c index 7a537cdfea..2daee85762 100644 --- a/src/os/src/detail/osDir.c +++ b/src/os/src/detail/osDir.c @@ -51,11 +51,6 @@ int taosMkDir(const char *path, mode_t mode) { } void taosRename(char* oldName, char *newName) { - if (0 == tsEnableVnodeBak) { - uInfo("vnode backup not enabled"); - return; - } - // if newName in not empty, rename return fail. // the newName must be empty or does not exist if (rename(oldName, newName)) { @@ -65,4 +60,48 @@ void taosRename(char* oldName, char *newName) { } } +void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays) { + DIR *dir = opendir(rootDir); + if (dir == NULL) return; + + int64_t ms = taosGetTimestampMs(); + struct dirent *de = NULL; + + while ((de = readdir(dir)) != NULL) { + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; + + char filename[1024]; + snprintf(filename, 1023, "%s/%s", rootDir, de->d_name); + if (de->d_type & DT_DIR) { + continue; + } else { + // struct stat fState; + // if (stat(fname, &fState) < 0) { + // continue; + // } + int32_t len = strlen(filename); + int64_t fileMs = 0; + for (int i = len - 1; i >= 0; ++i) { + if (filename[i] == '.') { + fileMs = atoll(filename + i + 1); + break; + } + } + + if (fileMs <= 0) continue; + int32_t days = (fileMs - ms) / 86400 + 1; + if (days > keepDays) { + (void)remove(filename); + uInfo("file:%s is removed, days:%d keepDays:%d", filename, days, keepDays); + + } else { + uTrace("file:%s won't be removed, days:%d keepDays:%d", filename, days, keepDays); + } + } + } + + closedir(dir); + rmdir(rootDir); +} + #endif diff --git a/src/os/src/linux/linuxEnv.c b/src/os/src/linux/linuxEnv.c index af3af97df2..14b40a1f18 100644 --- a/src/os/src/linux/linuxEnv.c +++ b/src/os/src/linux/linuxEnv.c @@ -25,7 +25,6 @@ void osInit() { } strcpy(tsDataDir, "/var/lib/power"); strcpy(tsLogDir, "/var/log/power"); - strcpy(tsLogbakDir, "/var/log/power/bak"); strcpy(tsScriptDir, "/etc/power"); #else if (configDir[0] == 0) { @@ -33,7 +32,6 @@ void osInit() { } strcpy(tsDataDir, "/var/lib/taos"); strcpy(tsLogDir, "/var/log/taos"); - strcpy(tsLogbakDir, "/var/log/taos/bak"); strcpy(tsScriptDir, "/etc/taos"); #endif diff --git a/src/os/src/windows/wEnv.c b/src/os/src/windows/wEnv.c index 2a758f31c3..8110a19490 100644 --- a/src/os/src/windows/wEnv.c +++ b/src/os/src/windows/wEnv.c @@ -30,7 +30,6 @@ void osInit() { strcpy(tsVnodeDir, "C:/PowerDB/data"); strcpy(tsDataDir, "C:/PowerDB/data"); strcpy(tsLogDir, "C:/PowerDB/log"); - strcpy(tsLogbakDir, "C:/PowerDB/log/bak"); strcpy(tsScriptDir, "C:/PowerDB/script"); #else @@ -41,7 +40,6 @@ void osInit() { strcpy(tsVnodeDir, "C:/TDengine/data"); strcpy(tsDataDir, "C:/TDengine/data"); strcpy(tsLogDir, "C:/TDengine/log"); - strcpy(tsLogbakDir, "C:/TDengine/log/bak"); strcpy(tsScriptDir, "C:/TDengine/script"); #endif diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index ab129b0ad4..b027304437 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -69,10 +69,8 @@ float tsAvailLogDirGB = 0; float tsMinimalLogDirGB = 0.1f; #ifdef _TD_POWER_ char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/power"; -char tsLogbakDir[TSDB_FILENAME_LEN] = "/var/log/power/bak"; #else char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/taos"; -char tsLogbakDir[TSDB_FILENAME_LEN] = "/var/log/taos/bak"; #endif static SLogObj tsLogObj = { .fileNum = 1 }; @@ -139,11 +137,25 @@ static void taosUnLockFile(int32_t fd) { } } +static void taosKeepOldLog(char *oldName) { + if (tsLogKeepDays <= 0) return; + + int64_t ms = taosGetTimestampMs(); + char fileName[LOG_FILE_NAME_LEN + 20]; + snprintf(fileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64, tsLogObj.logName, ms); + + uInfo("rename log file %s to %s", oldName, fileName); + taosRename(oldName, fileName); + taosRemoveOldLogFiles(tsLogDir, tsLogKeepDays); +} + static void *taosThreadToOpenNewFile(void *param) { - char name[LOG_FILE_NAME_LEN + 20]; + char keepName[LOG_FILE_NAME_LEN + 20]; + sprintf(keepName, "%s.%d", tsLogObj.logName, tsLogObj.flag); tsLogObj.flag ^= 1; tsLogObj.lines = 0; + char name[LOG_FILE_NAME_LEN + 20]; sprintf(name, "%s.%d", tsLogObj.logName, tsLogObj.flag); umask(0); @@ -153,6 +165,7 @@ static void *taosThreadToOpenNewFile(void *param) { uError("open new log file fail! fd:%d reason:%s", fd, strerror(errno)); return NULL; } + taosLockFile(fd); (void)lseek(fd, 0, SEEK_SET); @@ -160,9 +173,11 @@ static void *taosThreadToOpenNewFile(void *param) { tsLogObj.logHandle->fd = fd; tsLogObj.lines = 0; tsLogObj.openInProgress = 0; + taosCloseLogByFd(oldFd); uInfo("new log file is opened!!!"); - taosCloseLogByFd(oldFd); + taosKeepOldLog(keepName); + return NULL; } diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 5a778156ff..a463a2c90a 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -382,7 +382,13 @@ void vnodeRelease(void *pVnodeRaw) { char newDir[TSDB_FILENAME_LEN] = {0}; sprintf(rootDir, "%s/vnode%d", tsVnodeDir, vgId); sprintf(newDir, "%s/vnode%d", tsVnodeBakDir, vgId); - taosRename(rootDir, newDir); + + if (0 == tsEnableVnodeBak) { + vInfo("vgId:%d, vnode backup not enabled", pVnode->vgId); + } else { + taosRename(rootDir, newDir); + } + taosRemoveDir(rootDir); dnodeSendStatusMsgToMnode(); } From f65e3609767431931f6ce15badd2330b58197bc2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 8 Sep 2020 06:27:11 +0000 Subject: [PATCH 3/6] TD-1263 --- src/common/src/tglobal.c | 6 +++--- src/os/src/detail/osDir.c | 17 ++++++++--------- src/util/src/tlog.c | 37 +++++++++++++++++++++---------------- 3 files changed, 32 insertions(+), 28 deletions(-) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 5250d2947e..3888ec07a4 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -1013,18 +1013,18 @@ static void doInitGlobalConfig(void) { cfg.ptr = &tsNumOfLogLines; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 10000; + cfg.minValue = 1000; cfg.maxValue = 2000000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); cfg.option = "logKeepDays"; - cfg.ptr = &tsNumOfLogLines; + cfg.ptr = &tsLogKeepDays; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; - cfg.maxValue = 3650; + cfg.maxValue = 36500; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); diff --git a/src/os/src/detail/osDir.c b/src/os/src/detail/osDir.c index 2daee85762..ba0507f387 100644 --- a/src/os/src/detail/osDir.c +++ b/src/os/src/detail/osDir.c @@ -54,9 +54,9 @@ void taosRename(char* oldName, char *newName) { // if newName in not empty, rename return fail. // the newName must be empty or does not exist if (rename(oldName, newName)) { - uError("%s is modify to %s fail, reason:%s", oldName, newName, strerror(errno)); + uError("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); } else { - uInfo("%s is modify to %s success!", oldName, newName); + uInfo("successfully to rename file %s to %s", oldName, newName); } } @@ -64,7 +64,7 @@ void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays) { DIR *dir = opendir(rootDir); if (dir == NULL) return; - int64_t ms = taosGetTimestampMs(); + int64_t sec = taosGetTimestampSec(); struct dirent *de = NULL; while ((de = readdir(dir)) != NULL) { @@ -80,20 +80,19 @@ void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays) { // continue; // } int32_t len = strlen(filename); - int64_t fileMs = 0; - for (int i = len - 1; i >= 0; ++i) { + int64_t fileSec = 0; + for (int i = len - 1; i >= 0; i--) { if (filename[i] == '.') { - fileMs = atoll(filename + i + 1); + fileSec = atoll(filename + i + 1); break; } } - if (fileMs <= 0) continue; - int32_t days = (fileMs - ms) / 86400 + 1; + if (fileSec <= 100) continue; + int32_t days = ABS(sec - fileSec) / 86400 + 1; if (days > keepDays) { (void)remove(filename); uInfo("file:%s is removed, days:%d keepDays:%d", filename, days, keepDays); - } else { uTrace("file:%s won't be removed, days:%d keepDays:%d", filename, days, keepDays); } diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index b027304437..a8587de767 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -79,6 +79,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen static SLogBuff *taosLogBuffNew(int32_t bufSize); static void taosCloseLogByFd(int32_t oldFd); static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum); +extern void taosPrintGlobalCfg(); static int32_t taosStartLog() { pthread_attr_t threadAttr; @@ -140,11 +141,10 @@ static void taosUnLockFile(int32_t fd) { static void taosKeepOldLog(char *oldName) { if (tsLogKeepDays <= 0) return; - int64_t ms = taosGetTimestampMs(); + int64_t fileSec = taosGetTimestampSec(); char fileName[LOG_FILE_NAME_LEN + 20]; - snprintf(fileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64, tsLogObj.logName, ms); + snprintf(fileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64, tsLogObj.logName, fileSec); - uInfo("rename log file %s to %s", oldName, fileName); taosRename(oldName, fileName); taosRemoveOldLogFiles(tsLogDir, tsLogKeepDays); } @@ -174,8 +174,10 @@ static void *taosThreadToOpenNewFile(void *param) { tsLogObj.lines = 0; tsLogObj.openInProgress = 0; taosCloseLogByFd(oldFd); - uInfo("new log file is opened!!!"); - + + uInfo(" new log file:%d is opened", tsLogObj.flag); + uInfo("=================================="); + taosPrintGlobalCfg(); taosKeepOldLog(keepName); return NULL; @@ -282,20 +284,23 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { strcat(name, ".0"); } + if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) { + strcpy(name, fn); + strcat(name, ".1"); + } + + bool log0Exist = stat(name, &logstat0) >= 0; + bool log1Exist = stat(name, &logstat1) >= 0; + // if none of the log files exist, open 0, if both exists, open the old one - if (stat(name, &logstat0) < 0) { + if (!log0Exist && !log1Exist) { tsLogObj.flag = 0; + } else if (!log1Exist) { + tsLogObj.flag = 0; + } else if (!log0Exist) { + tsLogObj.flag = 1; } else { - if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) { - strcpy(name, fn); - strcat(name, ".1"); - } - - if (stat(name, &logstat1) < 0) { - tsLogObj.flag = 1; - } else { - tsLogObj.flag = (logstat0.st_mtime > logstat1.st_mtime) ? 0 : 1; - } + tsLogObj.flag = (logstat0.st_mtime > logstat1.st_mtime) ? 0 : 1; } char fileName[LOG_FILE_NAME_LEN + 50] = "\0"; From 159f59a15e738d2254905cac845a531483f62baf Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 8 Sep 2020 06:35:07 +0000 Subject: [PATCH 4/6] TD-1263 --- packaging/cfg/taos.cfg | 3 +++ src/common/src/tglobal.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index c12726d292..fdb8466706 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -189,6 +189,9 @@ # max number of rows per log filters # numOfLogLines 10000000 +# time of keeping log files, days +# logKeepDays 0 + # enable/disable async log # asyncLog 1 diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 3888ec07a4..2418375ee1 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -1024,7 +1024,7 @@ static void doInitGlobalConfig(void) { cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; - cfg.maxValue = 36500; + cfg.maxValue = 365000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); From bafad31882ba05ce3cd995400f44e7271568fb07 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 8 Sep 2020 15:18:16 +0800 Subject: [PATCH 5/6] TD-1263 compile in windows --- src/common/src/tglobal.c | 2 +- src/os/inc/osWindows.h | 2 -- src/os/src/detail/osDir.c | 8 ++------ src/os/src/windows/wDir.c | 31 ------------------------------- src/tsdb/src/tsdbFile.c | 2 +- 5 files changed, 4 insertions(+), 41 deletions(-) delete mode 100644 src/os/src/windows/wDir.c diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 2418375ee1..96e8fb26c6 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -254,7 +254,7 @@ bool taosCfgDynamicOptions(char *msg) { //if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue; if (cfg->valType != TAOS_CFG_VTYPE_INT32) continue; - int32_t cfgLen = strlen(cfg->option); + int32_t cfgLen = (int32_t)strlen(cfg->option); if (cfgLen != olen) continue; if (strncasecmp(option, cfg->option, olen) != 0) continue; *((int32_t *)cfg->ptr) = vint; diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index 224e41593d..ee17d2ddc3 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -58,8 +58,6 @@ extern "C" { int32_t BUILDIN_CTZL(uint64_t val); int32_t BUILDIN_CTZ(uint32_t val); -#define TAOS_OS_FUNC_DIR - #define TAOS_OS_FUNC_FILE #define TAOS_OS_FUNC_FILE_ISREG #define TAOS_OS_FUNC_FILE_ISDIR diff --git a/src/os/src/detail/osDir.c b/src/os/src/detail/osDir.c index ba0507f387..93651c78ef 100644 --- a/src/os/src/detail/osDir.c +++ b/src/os/src/detail/osDir.c @@ -18,8 +18,6 @@ #include "tglobal.h" #include "tulog.h" -#ifndef TAOS_OS_FUNC_DIR - void taosRemoveDir(char *rootDir) { DIR *dir = opendir(rootDir); if (dir == NULL) return; @@ -79,7 +77,7 @@ void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays) { // if (stat(fname, &fState) < 0) { // continue; // } - int32_t len = strlen(filename); + int32_t len = (int32_t)strlen(filename); int64_t fileSec = 0; for (int i = len - 1; i >= 0; i--) { if (filename[i] == '.') { @@ -89,7 +87,7 @@ void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays) { } if (fileSec <= 100) continue; - int32_t days = ABS(sec - fileSec) / 86400 + 1; + int32_t days = (int32_t)(ABS(sec - fileSec) / 86400 + 1); if (days > keepDays) { (void)remove(filename); uInfo("file:%s is removed, days:%d keepDays:%d", filename, days, keepDays); @@ -102,5 +100,3 @@ void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays) { closedir(dir); rmdir(rootDir); } - -#endif diff --git a/src/os/src/windows/wDir.c b/src/os/src/windows/wDir.c deleted file mode 100644 index c486cd0d40..0000000000 --- a/src/os/src/windows/wDir.c +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tulog.h" - -void taosRemoveDir(char *rootDir) { - uError("%s not implemented yet", __FUNCTION__); -} - -int taosMkDir(const char *path, mode_t mode) { - uError("%s not implemented yet", __FUNCTION__); - return 0; -} - -void taosMvDir(char* destDir, char *srcDir) { - uError("%s not implemented yet", __FUNCTION__); -} diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 4110c566d2..c7cd919ee8 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -576,5 +576,5 @@ static TSKEY tsdbGetCurrMinKey(int8_t precision, int32_t keep) { } static int tsdbGetCurrMinFid(int8_t precision, int32_t keep, int32_t days) { - return TSDB_KEY_FILEID(tsdbGetCurrMinKey(precision, keep), days, precision); + return (int)(TSDB_KEY_FILEID(tsdbGetCurrMinKey(precision, keep), days, precision)); } \ No newline at end of file From 227e8f37abeef8e1a1756df51c48e8d64f00f3e0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 8 Sep 2020 15:56:47 +0800 Subject: [PATCH 6/6] TD-1244 --- src/os/inc/osWindows.h | 1 + src/os/src/windows/wString.c | 25 +++++++++++++++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index ee17d2ddc3..d4f3d6d2af 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -39,6 +39,7 @@ #include #include #include +#include #include "msvcProcess.h" #include "msvcDirect.h" #include "msvcFcntl.h" diff --git a/src/os/src/windows/wString.c b/src/os/src/windows/wString.c index 0d9a28e288..1fb235a005 100644 --- a/src/os/src/windows/wString.c +++ b/src/os/src/windows/wString.c @@ -58,11 +58,20 @@ char *strsep(char **stringp, const char *delim) { char *getpass(const char *prefix) { static char passwd[TSDB_KEY_LEN] = {0}; - + memset(passwd, 0, TSDB_KEY_LEN); printf("%s", prefix); - scanf("%s", passwd); - char n = getchar(); + int32_t index = 0; + char ch; + while (index < TSDB_KEY_LEN) { + ch = getch(); + if (ch == '\n' || ch == '\r') { + break; + } else { + passwd[index++] = ch; + } + } + return passwd; } @@ -131,11 +140,11 @@ int tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int bytes) { } -/* Copy memory to memory until the specified number of bytes -has been copied, return pointer to following byte. -Overlap is NOT handled correctly. */ -void *mempcpy(void *dest, const void *src, size_t len) { - return (char*)memcpy(dest, src, len) + len; +/* Copy memory to memory until the specified number of bytes +has been copied, return pointer to following byte. +Overlap is NOT handled correctly. */ +void *mempcpy(void *dest, const void *src, size_t len) { + return (char*)memcpy(dest, src, len) + len; } /* Copy SRC to DEST, returning the address of the terminating '\0' in DEST. */