From 2028db36235b7d76a7ce448d29e448070f3bdb25 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 14 Nov 2024 10:00:07 +0800 Subject: [PATCH 01/21] enh: avoid invalid access of log file handle --- source/util/src/tlog.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index f70b145dbc..a0bf455cda 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -374,7 +374,7 @@ static OldFileKeeper *taosOpenNewFile() { } TdFilePtr pOldFile = tsLogObj.logHandle->pFile; - tsLogObj.logHandle->pFile = pFile; + atomic_store_ptr(&tsLogObj.logHandle->pFile, pFile); tsLogObj.lines = 0; tsLogObj.openInProgress = 0; OldFileKeeper *oldFileKeeper = taosMemoryMalloc(sizeof(OldFileKeeper)); @@ -666,7 +666,7 @@ static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *b if (tsAsyncLog) { TAOS_UNUSED(taosPushLogBuffer(tsLogObj.logHandle, buffer, len)); } else { - TAOS_UNUSED(taosWriteFile(tsLogObj.logHandle->pFile, buffer, len)); + TAOS_UNUSED(taosWriteFile(atomic_load_ptr(tsLogObj.logHandle->pFile), buffer, len)); } if (tsNumOfLogLines > 0) { From b4e6e2dfa81ef3dc597534c1809f0f42f4cfaee7 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 29 Nov 2024 13:40:33 +0800 Subject: [PATCH 02/21] enh: support specify log output by -o option --- include/util/tlog.h | 1 + include/util/tutil.h | 2 + source/client/src/clientEnv.c | 12 ++--- source/common/src/tglobal.c | 25 ++++++++++ source/dnode/mgmt/exe/dmMain.c | 29 ++++++++++- source/libs/function/src/udfd.c | 4 +- source/os/src/osDir.c | 4 +- source/os/src/osEnv.c | 2 +- source/util/src/tlog.c | 85 ++++++++++++++++++++++++++++++-- source/util/src/tutil.c | 27 ++++++++++ tools/shell/src/shellArguments.c | 26 +++++++++- 11 files changed, 199 insertions(+), 18 deletions(-) diff --git a/include/util/tlog.h b/include/util/tlog.h index a6c87593d1..451c2bb819 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -72,6 +72,7 @@ extern int32_t simDebugFlag; extern int32_t tqClientDebugFlag; +int32_t taosInitLogOutput(const char **ppLogName); int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc); void taosCloseLog(); void taosResetLog(); diff --git a/include/util/tutil.h b/include/util/tutil.h index aa3b774e84..c1b6b2505b 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -56,6 +56,8 @@ void taosIpPort2String(uint32_t ip, uint16_t port, char *str); void *tmemmem(const char *haystack, int hlen, const char *needle, int nlen); int32_t parseCfgReal(const char *str, float *out); +bool tIsValidFileName(const char *fileName, const char *pattern); +bool tIsValidFilePath(const char *filePath, const char *pattern); static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) { T_MD5_CTX context; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index b0be3a4d3b..117e0b65a9 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -41,6 +41,10 @@ #include "cus_name.h" #endif +#ifndef CUS_PROMPT +#define CUS_PROMPT "taos" +#endif + #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -954,12 +958,8 @@ void taos_init_imp(void) { taosHashSetFreeFp(appInfo.pInstMap, destroyAppInst); deltaToUtcInitOnce(); - char logDirName[64] = {0}; -#ifdef CUS_PROMPT - snprintf(logDirName, 64, "%slog", CUS_PROMPT); -#else - (void)snprintf(logDirName, 64, "taoslog"); -#endif + const char *logDirName = CUS_PROMPT "dlog"; + ENV_ERR_RET(taosInitLogOutput(&logDirName), "failed to init log output"); if (taosCreateLog(logDirName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { (void)printf(" WARING: Create %s failed:%s. configDir=%s\n", logDirName, strerror(errno), configDir); tscInitRes = -1; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 93c86a2dcd..2fc2b969cf 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -17,6 +17,7 @@ #include "tglobal.h" #include "defines.h" #include "os.h" +#include "osString.h" #include "tconfig.h" #include "tgrant.h" #include "tlog.h" @@ -1010,6 +1011,8 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { TAOS_RETURN(TSDB_CODE_SUCCESS); } +extern char *tsLogOutput; + static int32_t taosSetClientLogCfg(SConfig *pCfg) { SConfigItem *pItem = NULL; @@ -1017,6 +1020,26 @@ static int32_t taosSetClientLogCfg(SConfig *pCfg) { tstrncpy(tsLogDir, pItem->str, PATH_MAX); TAOS_CHECK_RETURN(taosExpandDir(tsLogDir, tsLogDir, PATH_MAX)); + if (tsLogOutput) { + char *pLog = tsLogOutput; + if (strcasecmp(pLog, "stdout") && strcasecmp(pLog, "stderr") && strcasecmp(pLog, "/dev/null")) { + char *pEnd = NULL; + if ((pEnd = strrchr(pLog, '/')) || (pEnd = strrchr(pLog, '\\'))) { + int32_t pathLen = POINTER_DISTANCE(pEnd, pLog) + 1; + if (*pLog == '/' || *pLog == '\\') { + tstrncpy(tsLogDir, pLog, TMIN(pathLen, PATH_MAX)); + } else { + int32_t len = strlen(tsLogDir); + if (tsLogDir[len - 1] != '/' && tsLogDir[len - 1] != '\\') { + tsLogDir[len++] = TD_DIRSEP_CHAR; + } + tstrncpy(tsLogDir + len, pLog, TMIN(PATH_MAX - len - 1, pathLen)); + } + cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true); + } + } + } + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "minimalLogDirGB"); tsLogSpace.reserved = (int64_t)(((double)pItem->fval) * 1024 * 1024 * 1024); @@ -1845,6 +1868,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile TAOS_CHECK_GOTO(taosAddSystemCfg(tsCfg), &lino, _exit); +#if 1 // duplicate operation since already loaded in taosCreateLog if ((code = taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) { (void)printf("failed to load cfg since %s\n", tstrerror(code)); cfgCleanup(tsCfg); @@ -1858,6 +1882,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile tsCfg = NULL; TAOS_RETURN(code); } +#endif if (tsc) { TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit); diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index ade5e16894..1ca2756708 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -49,6 +49,7 @@ #define DM_ENV_CMD "The env cmd variable string to use when configuring the server, such as: -e 'TAOS_FQDN=td1'." #define DM_ENV_FILE "The env variable file path to use when configuring the server, default is './.env', .env text can be 'TAOS_FQDN=td1'." #define DM_MACHINE_CODE "Get machine code." +#define DM_LOG_OUTPUT "Specify log output. Options:\n\r\t\t\t stdout, stderr, /dev/null, , /, \n\r\t\t\t * If OUTPUT contains an absolute directory, logs will be stored in that directory instead of logDir.\n\r\t\t\t * If OUTPUT contains a relative directory, logs will be stored in the directory combined with logDir and the relative directory." #define DM_VERSION "Print program version." #define DM_EMAIL "" #define DM_MEM_DBG "Enable memory debug" @@ -185,6 +186,7 @@ static void dmSetSignalHandle() { } extern bool generateNewMeta; +extern char *tsLogOutput; static int32_t dmParseArgs(int32_t argc, char const *argv[]) { global.startTime = taosGetTimestampMs(); @@ -239,6 +241,25 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { } } else if (strcmp(argv[i], "-k") == 0) { global.generateGrant = true; + } else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--log-output") == 0) { + if (i < argc - 1) { + if (strlen(argv[++i]) >= PATH_MAX) { + printf("failed to set log output since length overflow, max length is %d\n", PATH_MAX); + return TSDB_CODE_INVALID_CFG; + } + tsLogOutput = taosMemoryMalloc(PATH_MAX); + if (!tsLogOutput) { + printf("failed to set log output: '%s' since %s\n", argv[i], tstrerror(terrno)); + return terrno; + } + if (taosExpandDir(argv[i], tsLogOutput, PATH_MAX) != 0) { + printf("failed to expand log output: '%s' since %s\n", argv[i], tstrerror(terrno)); + return terrno; + } + } else { + printf("'-o' requires a parameter\n"); + return TSDB_CODE_INVALID_CFG; + } } else if (strcmp(argv[i], "-y") == 0) { global.generateCode = true; if (i < argc - 1) { @@ -316,6 +337,7 @@ static void dmPrintHelp() { printf("%s%s%s%s\n", indent, "-e,", indent, DM_ENV_CMD); printf("%s%s%s%s\n", indent, "-E,", indent, DM_ENV_FILE); printf("%s%s%s%s\n", indent, "-k,", indent, DM_MACHINE_CODE); + printf("%s%s%s%s\n", indent, "-o, --log-output=OUTPUT", indent, DM_LOG_OUTPUT); printf("%s%s%s%s\n", indent, "-y,", indent, DM_SET_ENCRYPTKEY); printf("%s%s%s%s\n", indent, "-dm,", indent, DM_MEM_DBG); printf("%s%s%s%s\n", indent, "-V,", indent, DM_VERSION); @@ -340,8 +362,11 @@ static int32_t dmCheckS3() { } static int32_t dmInitLog() { - return taosCreateLog(CUS_PROMPT "dlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, - 0); + const char *logName = CUS_PROMPT "dlog"; + + TAOS_CHECK_RETURN(taosInitLogOutput(&logName)); + + return taosCreateLog(logName, 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0); } static void taosCleanupArgs() { diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 0ee14f7820..bbfd43d5f7 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -1564,8 +1564,8 @@ static void udfdPrintVersion() { } static int32_t udfdInitLog() { - char logName[12] = {0}; - snprintf(logName, sizeof(logName), "%slog", "udfd"); + const char *logName = "udfdlog"; + TAOS_CHECK_RETURN(taosInitLogOutput(&logName)); return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, NULL, 0); } diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 777c6a9216..a6b0941577 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -339,7 +339,7 @@ int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) { OS_PARAM_CHECK(dirname); OS_PARAM_CHECK(outname); wordexp_t full_path; - int32_t code = wordexp(dirname, &full_path, 0); + int32_t code = wordexp(dirname, &full_path, 0); switch (code) { case 0: break; @@ -347,7 +347,7 @@ int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) { wordfree(&full_path); // FALL THROUGH default: - return code; + return terrno = TSDB_CODE_INVALID_PARA; } if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) { diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 41b34a9030..832deb574d 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -89,7 +89,7 @@ int32_t osDefaultInit() { } tstrncpy(tsDataDir, TD_DATA_DIR_PATH, sizeof(tsDataDir)); tstrncpy(tsLogDir, TD_LOG_DIR_PATH, sizeof(tsLogDir)); - if (strlen(tsTempDir) == 0){ + if (strlen(tsTempDir) == 0) { tstrncpy(tsTempDir, TD_TMP_DIR_PATH, sizeof(tsTempDir)); } diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index a0bf455cda..a5001d56c6 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -51,6 +51,13 @@ #define LOG_EDITION_FLG ("C") #endif +typedef enum { + LOG_OUTPUT_FILE = 0, // default + LOG_OUTPUT_STDOUT = 1, // stdout set by -o option on the command line + LOG_OUTPUT_STDERR = 2, // stderr set by -e option on the command line + LOG_OUTPUT_NULL = 4, // /dev/null set by -o option on the command line +} ELogOutputType; + typedef struct { char *buffer; int32_t buffStart; @@ -73,6 +80,7 @@ typedef struct { int32_t openInProgress; int64_t lastKeepFileSec; int64_t timestampToday; + int8_t outputType; // ELogOutputType pid_t pid; char logName[PATH_MAX]; char slowLogName[PATH_MAX]; @@ -96,6 +104,8 @@ bool tsAssert = true; #endif int32_t tsNumOfLogLines = 10000000; int32_t tsLogKeepDays = 0; + +char *tsLogOutput = NULL; LogFp tsLogFp = NULL; int64_t tsNumOfErrorLogs = 0; int64_t tsNumOfInfoLogs = 0; @@ -234,6 +244,58 @@ int32_t taosInitSlowLog() { return 0; } +int32_t taosInitLogOutput(const char **ppLogName) { + const char *pLog = tsLogOutput; + const char *pLogName = NULL; + if (pLog) { + if (!tIsValidFilePath(pLog, NULL)) { + fprintf(stderr, "invalid log output destination:%s, contains illegal char\n", pLog); + return TSDB_CODE_INVALID_CFG; + } + if (0 == strcasecmp(pLog, "stdout")) { + tsLogObj.outputType = LOG_OUTPUT_STDOUT; + if (ppLogName) *ppLogName = pLog; + return 0; + } + if (0 == strcasecmp(pLog, "stderr")) { + tsLogObj.outputType = LOG_OUTPUT_STDERR; + if (ppLogName) *ppLogName = pLog; + return 0; + } + if (0 == strcasecmp(pLog, "/dev/null")) { + tsLogObj.outputType = LOG_OUTPUT_NULL; + if (ppLogName) *ppLogName = pLog; + return 0; + } + int32_t len = strlen(pLog); + if (len < 1) { + fprintf(stderr, "invalid log output destination:%s, should not be empty\n", pLog); + return TSDB_CODE_INVALID_CFG; + } + const char *p = pLog + (len - 1); + if (*p == '/' || *p == '\\') { + return 0; + } + + if ((p = strrchr(pLog, '/')) || (p = strrchr(pLog, '\\'))) { + pLogName = p + 1; + } else { + pLogName = pLog; + } + if (strcmp(pLogName, ".") == 0 || strcmp(pLogName, "..") == 0) { + fprintf(stderr, "invalid log output destination:%s\n", pLog); + return TSDB_CODE_INVALID_CFG; + } + + if (!tIsValidFileName(pLogName, NULL)) { + fprintf(stderr, "invalid log output destination:%s, contains illegal char\n", pLog); + return TSDB_CODE_INVALID_CFG; + } + if (ppLogName) *ppLogName = pLogName; + } + return 0; +} + int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) { if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0; int32_t code = osUpdate(); @@ -241,6 +303,11 @@ int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) { uError("failed to update os info, reason:%s", tstrerror(code)); } + if (tsLogObj.outputType == LOG_OUTPUT_STDOUT || tsLogObj.outputType == LOG_OUTPUT_STDERR || + tsLogObj.outputType == LOG_OUTPUT_NULL) { + return 0; + } + TAOS_CHECK_RETURN(taosInitNormalLog(logName, maxFiles)); if (tsc) { TAOS_CHECK_RETURN(taosInitSlowLog()); @@ -283,6 +350,7 @@ void taosCloseLog() { taosMemoryFreeClear(tsLogObj.logHandle); tsLogObj.logHandle = NULL; } + taosMemoryFreeClear(tsLogOutput); } static bool taosLockLogFile(TdFilePtr pFile) { @@ -374,7 +442,7 @@ static OldFileKeeper *taosOpenNewFile() { } TdFilePtr pOldFile = tsLogObj.logHandle->pFile; - atomic_store_ptr(&tsLogObj.logHandle->pFile, pFile); + tsLogObj.logHandle->pFile = pFile; tsLogObj.lines = 0; tsLogObj.openInProgress = 0; OldFileKeeper *oldFileKeeper = taosMemoryMalloc(sizeof(OldFileKeeper)); @@ -666,7 +734,7 @@ static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *b if (tsAsyncLog) { TAOS_UNUSED(taosPushLogBuffer(tsLogObj.logHandle, buffer, len)); } else { - TAOS_UNUSED(taosWriteFile(atomic_load_ptr(tsLogObj.logHandle->pFile), buffer, len)); + TAOS_UNUSED(taosWriteFile(tsLogObj.logHandle->pFile, buffer, len)); } if (tsNumOfLogLines > 0) { @@ -680,10 +748,19 @@ static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *b } } - if (dflag & DEBUG_SCREEN) { + int fd = 0; + if (tsLogObj.outputType == LOG_OUTPUT_FILE) { + if (dflag & DEBUG_SCREEN) fd = 1; + } else if (tsLogObj.outputType == LOG_OUTPUT_STDOUT) { + fd = 1; + } else if (tsLogObj.outputType == LOG_OUTPUT_STDERR) { + fd = 2; + } + + if (fd) { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-result" - if (write(1, buffer, (uint32_t)len) < 0) { + if (write(fd, buffer, (uint32_t)len) < 0) { TAOS_UNUSED(printf("failed to write log to screen, reason:%s\n", strerror(errno))); } #pragma GCC diagnostic pop diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index 48338e7996..306b15909d 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "tutil.h" #include "tlog.h" +#include "regex.h" void *tmemmem(const char *haystack, int32_t hlen, const char *needle, int32_t nlen) { const char *limit; @@ -520,3 +521,29 @@ int32_t parseCfgReal(const char *str, float *out) { *out = val; return TSDB_CODE_SUCCESS; } + +bool tIsValidFileName(const char *fileName, const char *pattern) { + const char *fileNamePattern = "^[a-zA-Z0-9_.-]+$"; + + regex_t fileNameReg; + + if (pattern) fileNamePattern = pattern; + + if (regcomp(&fileNameReg, fileNamePattern, REG_EXTENDED) != 0) { + fprintf(stderr, "failed to compile file name pattern:%s\n", fileNamePattern); + return false; + } + + int32_t code = regexec(&fileNameReg, fileName, 0, NULL, 0); + regfree(&fileNameReg); + if (code != 0) { + return false; + } + return true; +} + +bool tIsValidFilePath(const char *filePath, const char *pattern) { + const char *filePathPattern = "^[a-zA-Z0-9/\\_.-]+$"; + + return tIsValidFileName(filePath, pattern ? pattern : filePathPattern); +} \ No newline at end of file diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 442329674d..390ee14ed3 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -49,7 +49,12 @@ #define SHELL_PKT_LEN "Packet length used for net test, default is 1024 bytes." #define SHELL_PKT_NUM "Packet numbers used for net test, default is 100." #define SHELL_BI_MODE "Set BI mode" -#define SHELL_VERSION "Print program version." +#define SHELL_LOG_OUTPUT \ + "Specify log output. Options:\n\r\t\t\t stdout, stderr, /dev/null, , /, " \ + "\n\r\t\t\t * If OUTPUT contains an absolute directory, logs will be stored in that directory " \ + "instead of logDir.\n\r\t\t\t * If OUTPUT contains a relative directory, logs will be stored in the directory " \ + "combined with logDir and the relative directory." +#define SHELL_VERSION "Print program version." #ifdef WEBSOCKET #define SHELL_DSN "Use dsn to connect to the cloud server or to a remote server which provides WebSocket connection." @@ -74,6 +79,7 @@ void shellPrintHelp() { printf("%s%s%s%s\r\n", indent, "-l,", indent, SHELL_PKT_LEN); printf("%s%s%s%s\r\n", indent, "-n,", indent, SHELL_NET_ROLE); printf("%s%s%s%s\r\n", indent, "-N,", indent, SHELL_PKT_NUM); + printf("%s%s%s%s\r\n", indent, "-o,", indent, SHELL_LOG_OUTPUT); printf("%s%s%s%s\r\n", indent, "-p,", indent, SHELL_PASSWORD); printf("%s%s%s%s\r\n", indent, "-P,", indent, SHELL_PORT); printf("%s%s%s%s\r\n", indent, "-r,", indent, SHELL_RAW_TIME); @@ -134,6 +140,7 @@ static struct argp_option shellOptions[] = { #endif {"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM}, {"bimode", 'B', 0, 0, SHELL_BI_MODE}, + {"log-output", 'o', "OUTPUT", 0, SHELL_LOG_OUTPUT}, {0}, }; @@ -152,6 +159,8 @@ static void shellParseArgsUseArgp(int argc, char *argv[]) { #define ARGP_ERR_UNKNOWN E2BIG #endif +extern char* tsLogOutput; + static int32_t shellParseSingleOpt(int32_t key, char *arg) { SShellArgs *pArgs = &shell.args; @@ -222,6 +231,21 @@ static int32_t shellParseSingleOpt(int32_t key, char *arg) { case 'N': pArgs->pktNum = atoi(arg); break; + case 'o': + if (strlen(arg) >= PATH_MAX) { + printf("failed to set log output since length overflow, max length is %d\n", PATH_MAX); + return TSDB_CODE_INVALID_CFG; + } + tsLogOutput = taosMemoryMalloc(PATH_MAX); + if (!tsLogOutput) { + printf("failed to set log output: '%s' since %s\n", arg, tstrerror(terrno)); + return terrno; + } + if (taosExpandDir(arg, tsLogOutput, PATH_MAX) != 0) { + printf("failed to expand log output: '%s' since %s\n", arg, tstrerror(terrno)); + return terrno; + } + break; #ifdef WEBSOCKET case 'R': pArgs->restful = true; From d256fb1c27611c58dae312d41e095b7478c9a1a5 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 29 Nov 2024 13:53:30 +0800 Subject: [PATCH 03/21] enh: support specify log output by -o option --- source/client/src/clientEnv.c | 10 +++++----- source/common/src/tglobal.c | 2 +- source/util/src/tlog.c | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 117e0b65a9..072cbee2f4 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -42,7 +42,7 @@ #endif #ifndef CUS_PROMPT -#define CUS_PROMPT "taos" +#define CUS_PROMPT "tao" #endif #define TSC_VAR_NOT_RELEASE 1 @@ -958,10 +958,10 @@ void taos_init_imp(void) { taosHashSetFreeFp(appInfo.pInstMap, destroyAppInst); deltaToUtcInitOnce(); - const char *logDirName = CUS_PROMPT "dlog"; - ENV_ERR_RET(taosInitLogOutput(&logDirName), "failed to init log output"); - if (taosCreateLog(logDirName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { - (void)printf(" WARING: Create %s failed:%s. configDir=%s\n", logDirName, strerror(errno), configDir); + const char *logName = CUS_PROMPT "slog"; + ENV_ERR_RET(taosInitLogOutput(&logName), "failed to init log output"); + if (taosCreateLog(logName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { + (void)printf(" WARING: Create %s failed:%s. configDir=%s\n", logName, strerror(errno), configDir); tscInitRes = -1; return; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 2fc2b969cf..18ff2af067 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1868,7 +1868,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile TAOS_CHECK_GOTO(taosAddSystemCfg(tsCfg), &lino, _exit); -#if 1 // duplicate operation since already loaded in taosCreateLog +#if 0 // duplicate operation since already loaded in taosCreateLog if ((code = taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) { (void)printf("failed to load cfg since %s\n", tstrerror(code)); cfgCleanup(tsCfg); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index a5001d56c6..4f2cce5869 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -54,7 +54,7 @@ typedef enum { LOG_OUTPUT_FILE = 0, // default LOG_OUTPUT_STDOUT = 1, // stdout set by -o option on the command line - LOG_OUTPUT_STDERR = 2, // stderr set by -e option on the command line + LOG_OUTPUT_STDERR = 2, // stderr set by -o option on the command line LOG_OUTPUT_NULL = 4, // /dev/null set by -o option on the command line } ELogOutputType; From a157caa38db8b8a4c39908d647c9b02a8e222fdc Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 29 Nov 2024 18:57:09 +0800 Subject: [PATCH 04/21] enh: process log rotate every 30 minutes --- include/common/cos_cp.h | 6 +- include/os/osFile.h | 4 +- source/common/src/cos.c | 8 +- source/common/src/cos_cp.c | 4 +- source/dnode/vnode/src/tsdb/tsdbRetention.c | 4 +- source/libs/wal/src/walMeta.c | 2 +- source/os/src/osDir.c | 6 +- source/os/src/osFile.c | 14 +-- source/util/src/tlog.c | 115 +++++++++++++++++++- 9 files changed, 133 insertions(+), 30 deletions(-) diff --git a/include/common/cos_cp.h b/include/common/cos_cp.h index 29532c0265..e54941f5d3 100644 --- a/include/common/cos_cp.h +++ b/include/common/cos_cp.h @@ -44,7 +44,7 @@ typedef struct { char file_path[TSDB_FILENAME_LEN]; // local file path int64_t file_size; // local file size, for upload - int32_t file_last_modified; // local file last modified time, for upload + int64_t file_last_modified; // local file last modified time, for upload char file_md5[64]; // md5 of the local file content, for upload, reserved char object_name[128]; // object name @@ -67,9 +67,9 @@ int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint); int32_t cos_cp_dump(SCheckpoint* checkpoint); void cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPart* parts, int64_t* consume_bytes); void cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag, uint64_t crc64); -void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int32_t mtime, +void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int64_t mtime, char const* upload_id, int64_t part_size); -bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int32_t mtime); +bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int64_t mtime); void cos_cp_build_download(SCheckpoint* checkpoint, char const* filepath, char const* object_name, int64_t object_size, char const* object_lmtime, char const* object_etag, int64_t part_size); diff --git a/include/os/osFile.h b/include/os/osFile.h index 536dee268a..1c397f3042 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -82,9 +82,9 @@ int32_t taosUnLockFile(TdFilePtr pFile); int32_t taosUmaskFile(int32_t maskVal); -int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *atime); +int32_t taosStatFile(const char *path, int64_t *size, int64_t *mtime, int64_t *atime); int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno); -int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime); +int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int64_t *mtime); bool taosCheckExistFile(const char *pathname); int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence); diff --git a/source/common/src/cos.c b/source/common/src/cos.c index a7e69ddc4c..de4f65f2ad 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -775,7 +775,7 @@ _exit: TAOS_RETURN(code); } -static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const char *file, int32_t lmtime, +static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const char *file, int64_t lmtime, char const *object_name, int64_t contentLength, S3PutProperties *put_prop, put_object_callback_data *data) { int32_t code = 0, lino = 0; @@ -963,7 +963,7 @@ _exit: int32_t s3PutObjectFromFile2ByEp(const char *file, const char *object_name, int8_t withcp, int8_t epIndex) { int32_t code = 0; - int32_t lmtime = 0; + int64_t lmtime = 0; const char *filename = 0; uint64_t contentLength = 0; const char *cacheControl = 0, *contentType = 0, *md5 = 0; @@ -1040,7 +1040,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t w static int32_t s3PutObjectFromFileOffsetByEp(const char *file, const char *object_name, int64_t offset, int64_t size, int8_t epIndex) { int32_t code = 0; - int32_t lmtime = 0; + int64_t lmtime = 0; const char *filename = 0; uint64_t contentLength = 0; const char *cacheControl = 0, *contentType = 0, *md5 = 0; @@ -1847,7 +1847,7 @@ _exit: typedef struct { int64_t size; - int32_t atime; + int64_t atime; char name[TSDB_FILENAME_LEN]; } SEvictFile; diff --git a/source/common/src/cos_cp.c b/source/common/src/cos_cp.c index 078b14c9e8..3469e8ecec 100644 --- a/source/common/src/cos_cp.c +++ b/source/common/src/cos_cp.c @@ -350,7 +350,7 @@ void cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag checkpoint->parts[part_index].crc64 = crc64; } -void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int32_t mtime, +void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int64_t mtime, char const* upload_id, int64_t part_size) { int i = 0; @@ -375,7 +375,7 @@ void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t static bool cos_cp_verify_md5(SCheckpoint* cp) { return true; } -bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int32_t mtime) { +bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int64_t mtime) { if (cos_cp_verify_md5(checkpoint) && checkpoint->file_size == size && checkpoint->file_last_modified == mtime) { return true; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 0072fd5e7f..c86a07584d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -660,7 +660,7 @@ static int32_t tsdbDoS3Migrate(SRTNer *rtner) { int32_t lcn = fobj->f->lcn; if (/*lcn < 1 && */ taosCheckExistFile(fobj->fname)) { - int32_t mtime = 0; + int64_t mtime = 0; int64_t size = 0; int32_t r = taosStatFile(fobj->fname, &size, &mtime, NULL); if (size > chunksize && mtime < rtner->now - tsS3UploadDelaySec) { @@ -687,7 +687,7 @@ static int32_t tsdbDoS3Migrate(SRTNer *rtner) { tsdbTFileLastChunkName(rtner->tsdb, fobj->f, fname1); if (taosCheckExistFile(fname1)) { - int32_t mtime = 0; + int64_t mtime = 0; int64_t size = 0; if (taosStatFile(fname1, &size, &mtime, NULL) != 0) { tsdbError("vgId:%d, %s failed at %s:%d ", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, __LINE__); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index b40a9eeefe..78f13a58ab 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -326,7 +326,7 @@ static int32_t walRepairLogFileTs(SWal* pWal, bool* updateMeta) { } walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); - int32_t mtime = 0; + int64_t mtime = 0; if (taosStatFile(fnameStr, NULL, &mtime, NULL) < 0) { wError("vgId:%d, failed to stat file due to %s, file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index a6b0941577..194e69c09c 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -301,14 +301,14 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue; char filename[1024]; - (void)snprintf(filename, sizeof(filename), "%s/%s", dirname, taosGetDirEntryName(de)); + (void)snprintf(filename, sizeof(filename), "%s%s%s", dirname, TD_DIRSEP, taosGetDirEntryName(de)); if (taosDirEntryIsDir(de)) { continue; } else { int32_t len = (int32_t)strlen(filename); if (len > 3 && strcmp(filename + len - 3, ".gz") == 0) { len -= 3; - }else{ + } else { continue; } @@ -324,7 +324,7 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { int32_t days = (int32_t)(TABS(sec - fileSec) / 86400 + 1); if (days > keepDays) { TAOS_UNUSED(taosRemoveFile(filename)); - uInfo("file:%s is removed, days:%d keepDays:%d, sed:%"PRId64, filename, days, keepDays, fileSec); + uInfo("file:%s is removed, days:%d keepDays:%d, sed:%" PRId64, filename, days, keepDays, fileSec); } else { // printf("file:%s won't be removed, days:%d keepDays:%d", filename, days, keepDays); } diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index c2484860ad..8ee0e74d4f 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -273,7 +273,7 @@ int32_t taosRenameFile(const char *oldName, const char *newName) { #endif } -int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *atime) { +int32_t taosStatFile(const char *path, int64_t *size, int64_t *mtime, int64_t *atime) { OS_PARAM_CHECK(path); #ifdef WINDOWS struct _stati64 fileStat; @@ -292,11 +292,11 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *a } if (mtime != NULL) { - *mtime = fileStat.st_mtime; + *mtime = fileStat.st_mtim.tv_sec; } if (atime != NULL) { - *atime = fileStat.st_atime; + *atime = fileStat.st_atim.tv_sec; } return 0; @@ -544,7 +544,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { return liOffset.QuadPart; } -int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { +int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int64_t *mtime) { if (pFile == NULL || pFile->hFile == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; @@ -571,7 +571,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { ULARGE_INTEGER ull; ull.LowPart = lastWriteTime.dwLowDateTime; ull.HighPart = lastWriteTime.dwHighDateTime; - *mtime = (int32_t)((ull.QuadPart - 116444736000000000ULL) / 10000000ULL); + *mtime = (int64_t)((ull.QuadPart - 116444736000000000ULL) / 10000000ULL); } return 0; } @@ -937,7 +937,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { return ret; } -int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { +int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int64_t *mtime) { if (pFile == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; @@ -960,7 +960,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { } if (mtime != NULL) { - *mtime = fileStat.st_mtime; + *mtime = fileStat.st_mtim.tv_sec; } return 0; diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 4f2cce5869..e364719cb1 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -94,6 +94,7 @@ static int8_t tsLogInited = 0; static SLogObj tsLogObj = {.fileNum = 1, .slowHandle = NULL}; static int64_t tsAsyncLogLostLines = 0; static int32_t tsDaylightActive; /* Currently in daylight saving time. */ +static SRWLatch tsLogRotateLatch = 0; bool tsLogEmbedded = 0; bool tsAsyncLog = true; @@ -408,10 +409,6 @@ static void taosKeepOldLog(char *oldName) { } } } - - if (tsLogKeepDays > 0) { - taosRemoveOldFiles(tsLogDir, tsLogKeepDays); - } } typedef struct { TdFilePtr pOldFile; @@ -460,11 +457,16 @@ static OldFileKeeper *taosOpenNewFile() { static void *taosThreadToCloseOldFile(void *param) { if (!param) return NULL; + taosWLockLatch(&tsLogRotateLatch); OldFileKeeper *oldFileKeeper = (OldFileKeeper *)param; taosSsleep(20); taosCloseLogByFd(oldFileKeeper->pOldFile); taosKeepOldLog(oldFileKeeper->keepName); taosMemoryFree(oldFileKeeper); + if (tsLogKeepDays > 0) { + taosRemoveOldFiles(tsLogDir, tsLogKeepDays); + } + taosWUnLockLatch(&tsLogRotateLatch); return NULL; } @@ -600,8 +602,8 @@ static void decideLogFileName(const char *fn, int32_t maxFileNum) { static void decideLogFileNameFlag() { char name[PATH_MAX + 50] = "\0"; - int32_t logstat0_mtime = 0; - int32_t logstat1_mtime = 0; + int64_t logstat0_mtime = 0; + int64_t logstat1_mtime = 0; bool log0Exist = false; bool log1Exist = false; @@ -1076,6 +1078,87 @@ static void taosWriteLog(SLogBuff *pLogBuf) { pLogBuf->writeInterval = 0; } +#define LOG_ROTATE_INTERVAL 30 +#ifndef LOG_ROTATE_BOOT +#define LOG_ROTATE_BOOT 3 +#endif +static void *taosLogRotateFunc(void *param) { + setThreadName("logRotate"); + int32_t code = 0; + taosWLockLatch(&tsLogRotateLatch); + // compress or remove the old log files + TdDirPtr pDir = taosOpenDir(tsLogDir); + if (!pDir) goto _exit; + TdDirEntryPtr de = NULL; + while ((de = taosReadDir(pDir))) { + if (taosDirEntryIsDir(de)) { + continue; + } + char *fname = taosGetDirEntryName(de); + if (!fname) { + continue; + } + + char *pSec = strrchr(fname, '.'); + if (!pSec) { + continue; + } + char *pIter = pSec; + bool isSec = true; + while (*(++pIter)) { + if (!isdigit(*pIter)) { + isSec = false; + break; + } + } + if (!isSec) { + continue; + } + + int64_t fileSec = 0; + if ((code = taosStr2int64(pSec + 1, NULL)) != 0) { + uWarn("%s:%d failed to convert %s to int64 since %s", __func__, __LINE__, pSec + 1, tstrerror(code)); + continue; + } + if (fileSec <= 100) { + continue; + } + + int64_t mtime = 0; + if ((code = taosStatFile(fname, NULL, &mtime, NULL)) != 0) { + uWarn("%s:%d failed to stat file %s since %s", __func__, __LINE__, fname, tstrerror(code)); + continue; + } + + int64_t elapseSec = taosGetTimestampMs() / 1000 - mtime; + + if (elapseSec < 86400) { + continue; + } + + char fullName[PATH_MAX] = {0}; + snprintf(fullName, sizeof(fullName), "%s%s%s", pDir, TD_DIRSEP, fname); + + int32_t days = elapseSec / 86400 + 1; + if (tsLogKeepDays > 0 && days > tsLogKeepDays) { + TAOS_UNUSED(taosRemoveFile(fullName)); + uInfo("file:%s is removed, days:%d keepDays:%d, sed:%" PRId64, fullName, days, tsLogKeepDays, fileSec); + } else { + taosKeepOldLog(fullName); // compress + } + } + if ((code = taosCloseDir(&pDir)) != 0) { + uWarn("%s:%d failed to close dir:%s since %s\n", __func__, __LINE__, tsLogDir, tstrerror(code)); + } + + if (tsLogKeepDays > 0) { + taosRemoveOldFiles(tsLogDir, tsLogKeepDays); + } +_exit: + taosWUnLockLatch(&tsLogRotateLatch); + return NULL; +} + static void *taosAsyncOutputLog(void *param) { SLogBuff *pLogBuf = (SLogBuff *)tsLogObj.logHandle; SLogBuff *pSlowBuf = (SLogBuff *)tsLogObj.slowHandle; @@ -1084,6 +1167,7 @@ static void *taosAsyncOutputLog(void *param) { int32_t count = 0; int32_t updateCron = 0; int32_t writeInterval = 0; + int32_t lastCheckMin = taosGetTimestampMs() / 60000 - (LOG_ROTATE_INTERVAL - LOG_ROTATE_BOOT); while (1) { if (pSlowBuf) { @@ -1109,6 +1193,25 @@ static void *taosAsyncOutputLog(void *param) { if (pSlowBuf) taosWriteSlowLog(pSlowBuf); break; } + + // process the log rotation every LOG_ROTATE_INTERVAL minutes + int32_t curMin = taosGetTimestampMs() / 60000; + if (curMin >= lastCheckMin) { + if ((curMin - lastCheckMin) >= LOG_ROTATE_INTERVAL) { + TdThread thread; + TdThreadAttr attr; + (void)taosThreadAttrInit(&attr); + (void)taosThreadAttrSetDetachState(&attr, PTHREAD_CREATE_DETACHED); + if (taosThreadCreate(&thread, &attr, taosLogRotateFunc, tsLogObj.logHandle) == 0) { + lastCheckMin = curMin; + } else { + uWarn("failed to create thread to process log buffer"); + } + (void)taosThreadAttrDestroy(&attr); + } + } else if (curMin < lastCheckMin) { + lastCheckMin = curMin; + } } return NULL; From 081eacb178762bfeba39521fddc1e2e93553d33d Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 30 Nov 2024 09:37:59 +0800 Subject: [PATCH 05/21] fix: ts log dir --- source/util/src/tlog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index e364719cb1..f08bfe96e0 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -1137,7 +1137,7 @@ static void *taosLogRotateFunc(void *param) { } char fullName[PATH_MAX] = {0}; - snprintf(fullName, sizeof(fullName), "%s%s%s", pDir, TD_DIRSEP, fname); + snprintf(fullName, sizeof(fullName), "%s%s%s", tsLogDir, TD_DIRSEP, fname); int32_t days = elapseSec / 86400 + 1; if (tsLogKeepDays > 0 && days > tsLogKeepDays) { From 9de36b0a19b868e28cc13a9664bd438112b5cd98 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 30 Nov 2024 22:53:45 +0800 Subject: [PATCH 06/21] fix: stat atime/mtime and check return of log output --- source/common/src/tglobal.c | 9 ++++++--- source/os/src/osFile.c | 6 +++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 18ff2af067..1507cbf14d 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1027,15 +1027,18 @@ static int32_t taosSetClientLogCfg(SConfig *pCfg) { if ((pEnd = strrchr(pLog, '/')) || (pEnd = strrchr(pLog, '\\'))) { int32_t pathLen = POINTER_DISTANCE(pEnd, pLog) + 1; if (*pLog == '/' || *pLog == '\\') { - tstrncpy(tsLogDir, pLog, TMIN(pathLen, PATH_MAX)); + if (pathLen >= PATH_MAX) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + tstrncpy(tsLogDir, pLog, pathLen); } else { int32_t len = strlen(tsLogDir); if (tsLogDir[len - 1] != '/' && tsLogDir[len - 1] != '\\') { tsLogDir[len++] = TD_DIRSEP_CHAR; } - tstrncpy(tsLogDir + len, pLog, TMIN(PATH_MAX - len - 1, pathLen)); + int32_t remain = PATH_MAX - len - 1; + if (remain < pathLen) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + tstrncpy(tsLogDir + len, pLog, pathLen); } - cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true); + TAOS_CHECK_RETURN(cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true)); } } } diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 8ee0e74d4f..486215c335 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -292,11 +292,11 @@ int32_t taosStatFile(const char *path, int64_t *size, int64_t *mtime, int64_t *a } if (mtime != NULL) { - *mtime = fileStat.st_mtim.tv_sec; + *mtime = fileStat.st_mtime; } if (atime != NULL) { - *atime = fileStat.st_atim.tv_sec; + *atime = fileStat.st_atime; } return 0; @@ -960,7 +960,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int64_t *mtime) { } if (mtime != NULL) { - *mtime = fileStat.st_mtim.tv_sec; + *mtime = fileStat.st_mtime; } return 0; From 01f10980b35e1f016a2fab659c30d311fdc03a25 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 30 Nov 2024 23:33:09 +0800 Subject: [PATCH 07/21] enh: log rotate --- source/util/src/tlog.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index f08bfe96e0..b441641c94 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -1078,9 +1078,9 @@ static void taosWriteLog(SLogBuff *pLogBuf) { pLogBuf->writeInterval = 0; } -#define LOG_ROTATE_INTERVAL 30 +#define LOG_ROTATE_INTERVAL 1800 // seconds #ifndef LOG_ROTATE_BOOT -#define LOG_ROTATE_BOOT 3 +#define LOG_ROTATE_BOOT 180 // seconds #endif static void *taosLogRotateFunc(void *param) { setThreadName("logRotate"); @@ -1132,7 +1132,7 @@ static void *taosLogRotateFunc(void *param) { int64_t elapseSec = taosGetTimestampMs() / 1000 - mtime; - if (elapseSec < 86400) { + if (elapseSec < 7200) { continue; } @@ -1167,7 +1167,7 @@ static void *taosAsyncOutputLog(void *param) { int32_t count = 0; int32_t updateCron = 0; int32_t writeInterval = 0; - int32_t lastCheckMin = taosGetTimestampMs() / 60000 - (LOG_ROTATE_INTERVAL - LOG_ROTATE_BOOT); + int64_t lastCheckSec = taosGetTimestampMs() / 1000 - (LOG_ROTATE_INTERVAL - LOG_ROTATE_BOOT); while (1) { if (pSlowBuf) { @@ -1195,22 +1195,22 @@ static void *taosAsyncOutputLog(void *param) { } // process the log rotation every LOG_ROTATE_INTERVAL minutes - int32_t curMin = taosGetTimestampMs() / 60000; - if (curMin >= lastCheckMin) { - if ((curMin - lastCheckMin) >= LOG_ROTATE_INTERVAL) { + int64_t curSec = taosGetTimestampMs() / 1000; + if (curSec >= lastCheckSec) { + if ((curSec - lastCheckSec) >= LOG_ROTATE_INTERVAL) { TdThread thread; TdThreadAttr attr; (void)taosThreadAttrInit(&attr); (void)taosThreadAttrSetDetachState(&attr, PTHREAD_CREATE_DETACHED); if (taosThreadCreate(&thread, &attr, taosLogRotateFunc, tsLogObj.logHandle) == 0) { - lastCheckMin = curMin; + lastCheckSec = curSec; } else { - uWarn("failed to create thread to process log buffer"); + uWarn("failed to create thread to process log rotation"); } (void)taosThreadAttrDestroy(&attr); } - } else if (curMin < lastCheckMin) { - lastCheckMin = curMin; + } else if (curSec < lastCheckSec) { + lastCheckSec = curSec; } } From 76aea0ddb9d2bf8524cddae4d9f998349ee4b25a Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 1 Dec 2024 13:12:32 +0800 Subject: [PATCH 08/21] enh: log rotate logic --- include/util/tlog.h | 2 +- source/common/src/tglobal.c | 28 +++------------------------- source/util/src/tlog.c | 24 ++++++++++++++++++++++++ 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/include/util/tlog.h b/include/util/tlog.h index 451c2bb819..9827331258 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -71,8 +71,8 @@ extern int32_t sndDebugFlag; extern int32_t simDebugFlag; extern int32_t tqClientDebugFlag; - int32_t taosInitLogOutput(const char **ppLogName); +int32_t taosSetLogOutput(); int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc); void taosCloseLog(); void taosResetLog(); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 1507cbf14d..20b3056af9 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1019,29 +1019,7 @@ static int32_t taosSetClientLogCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "logDir"); tstrncpy(tsLogDir, pItem->str, PATH_MAX); TAOS_CHECK_RETURN(taosExpandDir(tsLogDir, tsLogDir, PATH_MAX)); - - if (tsLogOutput) { - char *pLog = tsLogOutput; - if (strcasecmp(pLog, "stdout") && strcasecmp(pLog, "stderr") && strcasecmp(pLog, "/dev/null")) { - char *pEnd = NULL; - if ((pEnd = strrchr(pLog, '/')) || (pEnd = strrchr(pLog, '\\'))) { - int32_t pathLen = POINTER_DISTANCE(pEnd, pLog) + 1; - if (*pLog == '/' || *pLog == '\\') { - if (pathLen >= PATH_MAX) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); - tstrncpy(tsLogDir, pLog, pathLen); - } else { - int32_t len = strlen(tsLogDir); - if (tsLogDir[len - 1] != '/' && tsLogDir[len - 1] != '\\') { - tsLogDir[len++] = TD_DIRSEP_CHAR; - } - int32_t remain = PATH_MAX - len - 1; - if (remain < pathLen) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); - tstrncpy(tsLogDir + len, pLog, pathLen); - } - TAOS_CHECK_RETURN(cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true)); - } - } - } + TAOS_CHECK_RETURN(taosSetLogOutput(pCfg)); TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "minimalLogDirGB"); tsLogSpace.reserved = (int64_t)(((double)pItem->fval) * 1024 * 1024 * 1024); @@ -1871,7 +1849,6 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile TAOS_CHECK_GOTO(taosAddSystemCfg(tsCfg), &lino, _exit); -#if 0 // duplicate operation since already loaded in taosCreateLog if ((code = taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) { (void)printf("failed to load cfg since %s\n", tstrerror(code)); cfgCleanup(tsCfg); @@ -1885,11 +1862,12 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile tsCfg = NULL; TAOS_RETURN(code); } -#endif if (tsc) { + TAOS_CHECK_GOTO(taosSetClientLogCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit); } else { + TAOS_CHECK_GOTO(taosSetClientLogCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosUpdateServerCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosSetServerCfg(tsCfg), &lino, _exit); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index b441641c94..09835d7978 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -297,6 +297,30 @@ int32_t taosInitLogOutput(const char **ppLogName) { return 0; } +int32_t taosSetLogOutput(SConfig *pCfg) { + if (tsLogOutput && (tsLogObj.outputType == LOG_OUTPUT_FILE)) { + char *pLog = tsLogOutput; + char *pEnd = NULL; + if ((pEnd = strrchr(pLog, '/')) || (pEnd = strrchr(pLog, '\\'))) { + int32_t pathLen = POINTER_DISTANCE(pEnd, pLog) + 1; + if (*pLog == '/' || *pLog == '\\') { + if (pathLen >= PATH_MAX) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + tstrncpy(tsLogDir, pLog, pathLen); + } else { + int32_t len = strlen(tsLogDir); + if (tsLogDir[len - 1] != '/' && tsLogDir[len - 1] != '\\') { + tsLogDir[len++] = TD_DIRSEP_CHAR; + } + int32_t remain = PATH_MAX - len - 1; + if (remain < pathLen) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + tstrncpy(tsLogDir + len, pLog, pathLen); + } + TAOS_CHECK_RETURN(cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true)); + } + } + return 0; +} + int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) { if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0; int32_t code = osUpdate(); From 57afdc6612623192dab0650e095e659929f7051d Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 1 Dec 2024 14:51:12 +0800 Subject: [PATCH 09/21] fix: compile problem of log rotate on windows --- cmake/cmake.options | 4 ++++ include/util/tlog.h | 2 +- source/common/src/tglobal.c | 26 +++++++++++++++++++++++- source/dnode/mgmt/exe/dmMain.c | 1 - source/util/src/tlog.c | 34 ++++++-------------------------- tools/shell/src/shellArguments.c | 2 -- 6 files changed, 36 insertions(+), 33 deletions(-) diff --git a/cmake/cmake.options b/cmake/cmake.options index e3b5782d85..323132ea76 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -166,6 +166,10 @@ IF(${BUILD_WITH_ANALYSIS}) set(BUILD_WITH_S3 ON) ENDIF() +IF(${BUILD_TEST}) +add_definitions(-DBUILD_TEST) +ENDIF() + IF(${BUILD_S3}) IF(${BUILD_WITH_S3}) diff --git a/include/util/tlog.h b/include/util/tlog.h index 9827331258..78ca6ffe15 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -41,6 +41,7 @@ extern bool tsAsyncLog; extern bool tsAssert; extern int32_t tsNumOfLogLines; extern int32_t tsLogKeepDays; +extern char *tsLogOutput; extern LogFp tsLogFp; extern int64_t tsNumOfErrorLogs; extern int64_t tsNumOfInfoLogs; @@ -72,7 +73,6 @@ extern int32_t simDebugFlag; extern int32_t tqClientDebugFlag; int32_t taosInitLogOutput(const char **ppLogName); -int32_t taosSetLogOutput(); int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc); void taosCloseLog(); void taosResetLog(); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 20b3056af9..c1a3026394 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1011,7 +1011,31 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { TAOS_RETURN(TSDB_CODE_SUCCESS); } -extern char *tsLogOutput; +static int32_t taosSetLogOutput(SConfig *pCfg) { + if (tsLogOutput) { + char *pLog = tsLogOutput; + char *pEnd = NULL; + if (strcasecmp(pLog, "stdout") && strcasecmp(pLog, "stderr") && strcasecmp(pLog, "/dev/null")) { + if ((pEnd = strrchr(pLog, '/')) || (pEnd = strrchr(pLog, '\\'))) { + int32_t pathLen = POINTER_DISTANCE(pEnd, pLog) + 1; + if (*pLog == '/' || *pLog == '\\') { + if (pathLen >= PATH_MAX) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + tstrncpy(tsLogDir, pLog, pathLen); + } else { + int32_t len = strlen(tsLogDir); + if (tsLogDir[len - 1] != '/' && tsLogDir[len - 1] != '\\') { + tsLogDir[len++] = TD_DIRSEP_CHAR; + } + int32_t remain = PATH_MAX - len - 1; + if (remain < pathLen) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + tstrncpy(tsLogDir + len, pLog, pathLen); + } + TAOS_CHECK_RETURN(cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true)); + } + } + } + return 0; +} static int32_t taosSetClientLogCfg(SConfig *pCfg) { SConfigItem *pItem = NULL; diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 1ca2756708..e15e946d5a 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -186,7 +186,6 @@ static void dmSetSignalHandle() { } extern bool generateNewMeta; -extern char *tsLogOutput; static int32_t dmParseArgs(int32_t argc, char const *argv[]) { global.startTime = taosGetTimestampMs(); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 09835d7978..9678802a01 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -105,7 +105,6 @@ bool tsAssert = true; #endif int32_t tsNumOfLogLines = 10000000; int32_t tsLogKeepDays = 0; - char *tsLogOutput = NULL; LogFp tsLogFp = NULL; int64_t tsNumOfErrorLogs = 0; @@ -297,30 +296,6 @@ int32_t taosInitLogOutput(const char **ppLogName) { return 0; } -int32_t taosSetLogOutput(SConfig *pCfg) { - if (tsLogOutput && (tsLogObj.outputType == LOG_OUTPUT_FILE)) { - char *pLog = tsLogOutput; - char *pEnd = NULL; - if ((pEnd = strrchr(pLog, '/')) || (pEnd = strrchr(pLog, '\\'))) { - int32_t pathLen = POINTER_DISTANCE(pEnd, pLog) + 1; - if (*pLog == '/' || *pLog == '\\') { - if (pathLen >= PATH_MAX) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); - tstrncpy(tsLogDir, pLog, pathLen); - } else { - int32_t len = strlen(tsLogDir); - if (tsLogDir[len - 1] != '/' && tsLogDir[len - 1] != '\\') { - tsLogDir[len++] = TD_DIRSEP_CHAR; - } - int32_t remain = PATH_MAX - len - 1; - if (remain < pathLen) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); - tstrncpy(tsLogDir + len, pLog, pathLen); - } - TAOS_CHECK_RETURN(cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true)); - } - } - return 0; -} - int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) { if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0; int32_t code = osUpdate(); @@ -1102,10 +1077,13 @@ static void taosWriteLog(SLogBuff *pLogBuf) { pLogBuf->writeInterval = 0; } -#define LOG_ROTATE_INTERVAL 1800 // seconds -#ifndef LOG_ROTATE_BOOT -#define LOG_ROTATE_BOOT 180 // seconds +#define LOG_ROTATE_INTERVAL 1800 +#ifdef BUILD_TEST +#define LOG_ROTATE_BOOT 5 +#else +#define LOG_ROTATE_BOOT 180 #endif + static void *taosLogRotateFunc(void *param) { setThreadName("logRotate"); int32_t code = 0; diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 390ee14ed3..44686cc5f9 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -159,8 +159,6 @@ static void shellParseArgsUseArgp(int argc, char *argv[]) { #define ARGP_ERR_UNKNOWN E2BIG #endif -extern char* tsLogOutput; - static int32_t shellParseSingleOpt(int32_t key, char *arg) { SShellArgs *pArgs = &shell.args; From 09e1abc9677d516eff3fe27be36f31b29ea3ea06 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 1 Dec 2024 18:12:16 +0800 Subject: [PATCH 10/21] test: add test case for log output option --- tests/system-test/0-others/taosdlog.py | 169 +++++++++++++++---------- 1 file changed, 103 insertions(+), 66 deletions(-) diff --git a/tests/system-test/0-others/taosdlog.py b/tests/system-test/0-others/taosdlog.py index d4698960cd..604617087f 100644 --- a/tests/system-test/0-others/taosdlog.py +++ b/tests/system-test/0-others/taosdlog.py @@ -1,66 +1,103 @@ -import taos -import sys -import time -import os - -from util.log import * -from util.sql import * -from util.cases import * -from util.dnodes import * - -class TDTestCase: - - def init(self, conn, logSql, replicaVar=1): - self.replicaVar = int(replicaVar) - tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor()) - - def getBuildPath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files or "taosd.exe" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root) - len("/build/bin")] - break - return buildPath - - def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring - tdSql.prepare() - # time.sleep(2) - tdSql.query("create user testpy pass 'testpy'") - - buildPath = self.getBuildPath() - if (buildPath == ""): - tdLog.exit("taosd not found!") - else: - tdLog.info("taosd found in %s" % buildPath) - logPath = buildPath + "/../sim/dnode1/log" - tdLog.info("log path: %s" % logPath) - - tdDnodes.stop(1) - time.sleep(2) - tdSql.error("select * from information_schema.ins_databases") - os.system("rm -rf %s" % logPath) - if os.path.exists(logPath) == True: - tdLog.exit("log pat still exist!") - - tdDnodes.start(1) - time.sleep(2) - if os.path.exists(logPath) != True: - tdLog.exit("log pat is not generated!") - - tdSql.query("select * from information_schema.ins_databases") - - def stop(self): - tdSql.close() - tdLog.success(f"{__file__} successfully executed") - -tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + tdSql.prepare() + self.buildPath = self.getBuildPath() + if (self.buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % self.buildPath) + self.logPath = self.buildPath + "/../sim/dnode1/log" + tdLog.info("log path: %s" % self.logPath) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def logPathBasic(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.query("create user testpy pass 'testpy'") + + tdDnodes.stop(1) + time.sleep(2) + tdSql.error("select * from information_schema.ins_databases") + tdSql.checkRows(2) + os.system("rm -rf %s" % self.logPath) + if os.path.exists(self.logPath) == True: + tdLog.exit("log path still exist!") + + tdDnodes.start(1) + time.sleep(2) + if os.path.exists(self.logPath) != True: + tdLog.exit("log path is not generated!") + + tdSql.query("select * from information_schema.ins_databases") + tdSql.checkRows(2) + + def prepareCfg(self, cfgPath, cfgDict): + with open(cfgPath + "/taos.cfg", "w") as f: + for key in cfgDict: + f.write("%s %s\n" % (key, cfgDict[key])) + + def check_function_1(self): + # Implementation of check function 1 + tdLog.info("Running check function 1") + # Add your check logic here + + def check_function_2(self): + # Implementation of check function 2 + tdLog.info("Running check function 2") + # Add your check logic here + + def check_function_3(self): + # Implementation of check function 3 + tdLog.info("Running check function 3") + # Add your check logic here + + def prepareCheckFunctions(self): + self.check_functions = { + "check_function_1": self.check_function_1, + "check_function_2": self.check_function_2, + "check_function_3": self.check_function_3 + } + + def checkLogOutput(self): + self.prepareCheckFunctions() + for key, check_func in self.check_functions.items(): + print(f"Running {key}") + check_func() + + def run(self): + # self.logPathBasic() + self.checkLogOutput() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 1e8a2ab8392d8914adf800d3ee0ecac134aa5ccc Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 3 Dec 2024 07:03:53 +0800 Subject: [PATCH 11/21] test: add test case for log output --- source/common/src/tglobal.c | 3 + source/dnode/mgmt/exe/dmMain.c | 22 +- source/util/src/tlog.c | 18 +- tests/parallel_test/cases.task | 1 + tests/system-test/0-others/taosdlog.py | 304 +++++++++++++++++++++---- 5 files changed, 292 insertions(+), 56 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c1a3026394..62a1e3f500 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1032,6 +1032,9 @@ static int32_t taosSetLogOutput(SConfig *pCfg) { } TAOS_CHECK_RETURN(cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true)); } + } else { + tstrncpy(tsLogDir, pLog, PATH_MAX); + TAOS_CHECK_RETURN(cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true)); } } return 0; diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index e15e946d5a..d270699e58 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -240,23 +240,28 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { } } else if (strcmp(argv[i], "-k") == 0) { global.generateGrant = true; - } else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--log-output") == 0) { - if (i < argc - 1) { - if (strlen(argv[++i]) >= PATH_MAX) { - printf("failed to set log output since length overflow, max length is %d\n", PATH_MAX); + } else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--log-output") == 0 || + strncmp(argv[i], "--log-output=", 13) == 0) { + if ((i < argc - 1) || ((i == argc - 1) && strncmp(argv[i], "--log-output=", 13) == 0)) { + int32_t klen = strlen(argv[i]); + int32_t vlen = klen < 13 ? strlen(argv[++i]) : klen - 13; + const char *val = argv[i]; + if (klen >= 13) val += 13; + if (vlen <= 0 || vlen >= PATH_MAX) { + printf("failed to set log output since invalid vlen:%d, valid range: [1, %d)\n", vlen, PATH_MAX); return TSDB_CODE_INVALID_CFG; } tsLogOutput = taosMemoryMalloc(PATH_MAX); if (!tsLogOutput) { - printf("failed to set log output: '%s' since %s\n", argv[i], tstrerror(terrno)); + printf("failed to set log output: '%s' since %s\n", val, tstrerror(terrno)); return terrno; } - if (taosExpandDir(argv[i], tsLogOutput, PATH_MAX) != 0) { - printf("failed to expand log output: '%s' since %s\n", argv[i], tstrerror(terrno)); + if (taosExpandDir(val, tsLogOutput, PATH_MAX) != 0) { + printf("failed to expand log output: '%s' since %s\n", val, tstrerror(terrno)); return terrno; } } else { - printf("'-o' requires a parameter\n"); + printf("'%s' requires a parameter\n", argv[i]); return TSDB_CODE_INVALID_CFG; } } else if (strcmp(argv[i], "-y") == 0) { @@ -402,6 +407,7 @@ int mainWindows(int argc, char **argv) { int32_t code = 0; #endif + char *p = taosMemoryCalloc(10, 10); if (global.generateGrant) { dmGenerateGrant(); taosCleanupArgs(); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 9678802a01..707923f200 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -1078,10 +1078,12 @@ static void taosWriteLog(SLogBuff *pLogBuf) { } #define LOG_ROTATE_INTERVAL 1800 -#ifdef BUILD_TEST -#define LOG_ROTATE_BOOT 5 +#ifdef ASSERT_NOT_CORE +#define LOG_INACTIVE_TIME 3600 +#define LOG_ROTATE_BOOT 5 #else -#define LOG_ROTATE_BOOT 180 +#define LOG_INACTIVE_TIME 5 +#define LOG_ROTATE_BOOT (LOG_INACTIVE_TIME + 1) #endif static void *taosLogRotateFunc(void *param) { @@ -1118,7 +1120,7 @@ static void *taosLogRotateFunc(void *param) { } int64_t fileSec = 0; - if ((code = taosStr2int64(pSec + 1, NULL)) != 0) { + if ((code = taosStr2int64(pSec + 1, &fileSec)) != 0) { uWarn("%s:%d failed to convert %s to int64 since %s", __func__, __LINE__, pSec + 1, tstrerror(code)); continue; } @@ -1132,19 +1134,19 @@ static void *taosLogRotateFunc(void *param) { continue; } - int64_t elapseSec = taosGetTimestampMs() / 1000 - mtime; + int64_t inactiveSec = taosGetTimestampMs() / 1000 - mtime; - if (elapseSec < 7200) { + if (inactiveSec < LOG_INACTIVE_TIME) { continue; } char fullName[PATH_MAX] = {0}; snprintf(fullName, sizeof(fullName), "%s%s%s", tsLogDir, TD_DIRSEP, fname); - int32_t days = elapseSec / 86400 + 1; + int32_t days = inactiveSec / 86400 + 1; if (tsLogKeepDays > 0 && days > tsLogKeepDays) { TAOS_UNUSED(taosRemoveFile(fullName)); - uInfo("file:%s is removed, days:%d keepDays:%d, sed:%" PRId64, fullName, days, tsLogKeepDays, fileSec); + uInfo("file:%s is removed, days:%d, keepDays:%d, sed:%" PRId64, fullName, days, tsLogKeepDays, fileSec); } else { taosKeepOldLog(fullName); // compress } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index db0339381e..ac33c38545 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -352,6 +352,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/telemetry.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/backquote_check.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdMonitor.py +,,n,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdlog.py ,,n,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 ,,n,system-test,python3 ./test.py -f 0-others/udfTest.py ,,n,system-test,python3 ./test.py -f 0-others/udf_create.py diff --git a/tests/system-test/0-others/taosdlog.py b/tests/system-test/0-others/taosdlog.py index 604617087f..a0981d7a72 100644 --- a/tests/system-test/0-others/taosdlog.py +++ b/tests/system-test/0-others/taosdlog.py @@ -2,11 +2,18 @@ import taos import sys import time import os +import os.path +import platform +import concurrent.futures +import shutil # Add this import from util.log import * from util.sql import * from util.cases import * from util.dnodes import * +import time +import socket +import subprocess class TDTestCase: @@ -20,8 +27,13 @@ class TDTestCase: tdLog.exit("taosd not found!") else: tdLog.info("taosd found in %s" % self.buildPath) + self.taosdPath = self.buildPath + "/build/bin/taosd" + self.taosPath = self.buildPath + "/build/bin/taos" self.logPath = self.buildPath + "/../sim/dnode1/log" + tdLog.info("taosd path: %s" % self.taosdPath) + tdLog.info("taos path: %s" % self.taosPath) tdLog.info("log path: %s" % self.logPath) + self.commonCfgDict = {} def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -39,60 +51,272 @@ class TDTestCase: break return buildPath - def logPathBasic(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring - tdSql.query("create user testpy pass 'testpy'") - - tdDnodes.stop(1) - time.sleep(2) - tdSql.error("select * from information_schema.ins_databases") - tdSql.checkRows(2) - os.system("rm -rf %s" % self.logPath) - if os.path.exists(self.logPath) == True: - tdLog.exit("log path still exist!") - - tdDnodes.start(1) - time.sleep(2) - if os.path.exists(self.logPath) != True: - tdLog.exit("log path is not generated!") - - tdSql.query("select * from information_schema.ins_databases") - tdSql.checkRows(2) - def prepareCfg(self, cfgPath, cfgDict): + tdLog.info("make dir %s" % cfgPath) + os.makedirs(cfgPath, exist_ok=True) with open(cfgPath + "/taos.cfg", "w") as f: + for key in self.commonCfgDict: + f.write("%s %s\n" % (key, self.commonCfgDict[key])) for key in cfgDict: f.write("%s %s\n" % (key, cfgDict[key])) + if not os.path.exists(cfgPath + "/taos.cfg"): + tdLog.exit("taos.cfg not found in %s" % cfgPath) + else: + tdLog.info("taos.cfg found in %s" % cfgPath) - def check_function_1(self): - # Implementation of check function 1 - tdLog.info("Running check function 1") - # Add your check logic here + def checkProcessPid(self,processName): + i=0 + while i<60: + tdLog.info(f"wait stop {processName}") + processPid = subprocess.getstatusoutput(f'ps aux|grep {processName} |grep -v "grep"|awk \'{{print $2}}\'')[1] + tdLog.info(f"times:{i},{processName}-pid:{processPid}") + if(processPid == ""): + break + i += 1 + time.sleep(1) + else: + tdLog.info(f'this processName is not stoped in 60s') - def check_function_2(self): - # Implementation of check function 2 - tdLog.info("Running check function 2") - # Add your check logic here + def closeTaosd(self, signal=9): + tdLog.info("Closing taosd") + if platform.system().lower() == 'windows': + psCmd = "for /f %%a in ('wmic process where \"name='taosd.exe' and CommandLine like '%%dnode%d%%'\" get processId ^| xargs echo ^| awk ^'{print $2}^' ^&^& echo aa') do @(ps | grep %%a | awk '{print $1}' | xargs)" % (self.index) + else: + psCmd = "ps -ef | grep -w taosd | grep -v grep | awk '{print $2}' | xargs" + tdLog.info(f"psCmd:{psCmd}") - def check_function_3(self): - # Implementation of check function 3 - tdLog.info("Running check function 3") - # Add your check logic here + try: + rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + processID = rem.stdout.decode().strip() + tdLog.info(f"processID:{processID}") + except Exception as e: + tdLog.info(f"closeTaosd error:{e}") + processID = "" + tdLog.info(f"processID:{processID}") + onlyKillOnceWindows = 0 + while(processID): + if not platform.system().lower() == 'windows' or (onlyKillOnceWindows == 0 and platform.system().lower() == 'windows'): + killCmd = "kill -%d %s > /dev/null 2>&1" % (signal, processID) + if platform.system().lower() == 'windows': + killCmd = "kill -%d %s > nul 2>&1" % (signal, processID) + tdLog.info(f"killCmd:{killCmd}") + os.system(killCmd) + tdLog.info(f"killed taosd process {processID}") + onlyKillOnceWindows = 1 + time.sleep(1) + try: + rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + processID = rem.stdout.decode().strip() + except Exception as e: + tdLog.info(f"closeTaosd error:{e}") + processID = "" - def prepareCheckFunctions(self): + def closeTaos(self, signal=9): + tdLog.info("Closing taos") + if platform.system().lower() == 'windows': + psCmd = "for /f %%a in ('wmic process where \"name='taos.exe'\" get processId ^| xargs echo ^| awk ^'{print $2}^' ^&^& echo aa') do @(ps | grep %%a | awk '{print $1}' | xargs)" + else: + psCmd = "ps -ef | grep -w taos | grep -v grep | awk '{print $2}'" + tdLog.info(f"psCmd:{psCmd}") + + try: + rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + processID = rem.stdout.decode().strip() + tdLog.info(f"processID:{processID}") + except Exception as e: + tdLog.info(f"closeTaos error:{e}") + processID = "" + tdLog.info(f"processID:{processID}") + onlyKillOnceWindows = 0 + while(processID): + if not platform.system().lower() == 'windows' or (onlyKillOnceWindows == 0 and platform.system().lower() == 'windows'): + killCmd = "kill -%d %s > /dev/null 2>&1" % (signal, processID) + if platform.system().lower() == 'windows': + killCmd = "kill -%d %s > nul 2>&1" % (signal, processID) + tdLog.info(f"killCmd:{killCmd}") + os.system(killCmd) + tdLog.info(f"killed taos process {processID}") + onlyKillOnceWindows = 1 + time.sleep(1) + try: + rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + processID = rem.stdout.decode().strip() + except Exception as e: + tdLog.info(f"closeTaos error:{e}") + processID = "" + + def openBin(self, binPath, waitSec=5): + tdLog.info(f"Opening {binPath}") + try: + process = subprocess.Popen(binPath, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + time.sleep(waitSec) + if process.poll() is None: + tdLog.info(f"{binPath} started successfully") + else: + error = process.stderr.read().decode(encoding="utf-8").strip() + raise Exception(f"Failed to start {binPath}: {error}") + except Exception as e: + raise Exception(f"Failed to start {binPath}: %s" % repr(e)) + + def openTaosd(self, args="", waitSec=8): + self.openBin(f'{self.taosdPath} {args}', waitSec) + + def openTaos(self, args="", waitSec=3): + self.openBin(f'{self.taosPath} {args}', waitSec) + + + def prepare_logoutput(self, desc, port, logOutput): + tdLog.info("Preparing %s, port:%s" % (desc, port)) + dnodePath = self.buildPath + "/../sim/dnode%s" % port + tdLog.info("remove dnodePath:%s" % dnodePath) + try: + shutil.rmtree(dnodePath) + except Exception as e: + tdLog.info(f"Failed to remove directory {dnodePath}: {e}") + try: + self.prepareCfg(dnodePath, {"serverPort": port, + "dataDir": dnodePath + os.sep + "data", + "logDir": dnodePath + os.sep + "log"}) + except Exception as e: + tdLog.info(f"Failed to prepare configuration for {dnodePath}: {e}") + return + try: + self.openTaosd(f"-c {dnodePath} {logOutput}") + self.openTaos(f"-c {dnodePath} {logOutput}") + except Exception as e: + tdLog.info(f"Failed to open taosd or taos for {dnodePath}: {e}") + + def prepare_stdout(self): + list = self.prepare_list[0] + self.prepare_logoutput(list[0], list[1], "-o " + list[0]) + + def check_stdout(self): + tdLog.info("Running check stdout") + dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[0][1] + tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0")) + tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0")) + + def prepare_stderr(self): + list = self.prepare_list[1] + self.prepare_logoutput(list[0], list[1], "--log-output " + list[0]) + + def check_stderr(self): + tdLog.info("Running check stderr") + dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[1][1] + tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0")) + tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0")) + + def prepare_dev_null(self): + list = self.prepare_list[2] + self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0]) + + def check_dev_null(self): + tdLog.info("Running check /dev/null") + dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[2][1] + tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0")) + tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0")) + + def prepare_fullpath(self): + list = self.prepare_list[3] + dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[3][1] + self.prepare_logoutput(list[0], list[1], "-o " + dnodePath + "/log0/" ) + + def check_fullpath(self): + tdLog.info("Running check fullpath") + logPath = self.buildPath + "/../sim/dnode%s/log0/" % self.prepare_list[3][1] + tdSql.checkEqual(True, os.path.exists(f"{logPath}taosdlog.0")) + tdSql.checkEqual(True, os.path.exists(f"{logPath}taoslog0.0")) + + def prepare_fullname(self): + list = self.prepare_list[4] + dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[4][1] + self.prepare_logoutput(list[0], list[1], "--log-output " + dnodePath + "/log0/" + list[0]) + dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[4][1] + + def check_fullname(self): + tdLog.info("Running check fullname") + logPath = self.buildPath + "/../sim/dnode%s/log0/" % self.prepare_list[4][1] + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[4][0] + ".0")) + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[4][0] + "0.0")) + + def prepare_relativepath(self): + list = self.prepare_list[5] + self.prepare_logoutput(list[0], list[1], "--log-output=" + "log0/") + + def check_relativepath(self): + tdLog.info("Running check relativepath") + logPath = self.buildPath + "/../sim/dnode%s/log/log0/" % self.prepare_list[5][1] + tdSql.checkEqual(True, os.path.exists(logPath + "taosdlog.0")) + tdSql.checkEqual(True, os.path.exists(logPath + "taoslog0.0")) + + def prepare_relativename(self): + list = self.prepare_list[6] + self.prepare_logoutput(list[0], list[1], "-o " + "log0/" + list[0]) + def check_relativename(self): + tdLog.info("Running check relativename") + logPath = self.buildPath + "/../sim/dnode%s/log/log0/" % self.prepare_list[6][1] + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[6][0] + ".0")) + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[6][0] + "0.0")) + + def prepare_filename(self): + list = self.prepare_list[7] + self.prepare_logoutput(list[0], list[1], "--log-output " + list[0]) + def check_filename(self): + tdLog.info("Running check filename") + logPath = self.buildPath + "/../sim/dnode%s/log/" % self.prepare_list[7][1] + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[7][0] + ".0")) + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[7][0] + "0.0")) + + def prepare_empty(self): + list = self.prepare_list[8] + self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0]) + def check_empty(self): + tdLog.info("Running check empty") + logPath = self.buildPath + "/../sim/dnode%s/log" % self.prepare_list[8][1] + tdSql.checkEqual(False, os.path.exists(f"{logPath}/taosdlog.0")) + tdSql.checkEqual(False, os.path.exists(f"{logPath}/taoslog.0")) + + def prepareCheckResources(self): + self.prepare_list = [["stdout", "10030"], ["stderr", "10031"], ["/dev/null", "10032"], + ["fullpath", "10033"], ["fullname", "10034"], ["relativepath", "10035"], + ["relativename", "10036"], ["filename", "10037"], ["empty", "10038"]] self.check_functions = { - "check_function_1": self.check_function_1, - "check_function_2": self.check_function_2, - "check_function_3": self.check_function_3 + self.prepare_stdout: self.check_stdout, + self.prepare_stderr: self.check_stderr, + self.prepare_dev_null: self.check_dev_null, + self.prepare_fullpath: self.check_fullpath, + self.prepare_fullname: self.check_fullname, + self.prepare_relativepath: self.check_relativepath, + self.prepare_relativename: self.check_relativename, + self.prepare_filename: self.check_filename, + self.prepare_empty: self.check_empty, } def checkLogOutput(self): - self.prepareCheckFunctions() - for key, check_func in self.check_functions.items(): - print(f"Running {key}") - check_func() + self.closeTaosd() + self.closeTaos() + self.prepareCheckResources() + with concurrent.futures.ThreadPoolExecutor() as executor: + prepare_futures = [executor.submit(prepare_func) for prepare_func, _ in self.check_functions.items()] + for future in concurrent.futures.as_completed(prepare_futures): + try: + future.result() + except Exception as e: + raise Exception(f"Error in prepare function: {e}") + + check_futures = [executor.submit(check_func) for _, check_func in self.check_functions.items()] + for future in concurrent.futures.as_completed(check_futures): + try: + future.result() + except Exception as e: + raise Exception(f"Error in prepare function: {e}") + self.closeTaosd() + self.closeTaos() + + def is_windows(self): + return os.name == 'nt' def run(self): - # self.logPathBasic() self.checkLogOutput() def stop(self): From 60450f3e24fa505bed6d9e763f18f11421b366ab Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 3 Dec 2024 11:13:14 +0800 Subject: [PATCH 12/21] test: test case for log output and rotate --- source/dnode/mgmt/exe/dmMain.c | 1 - source/util/src/tlog.c | 19 +- tests/system-test/0-others/taosdlog.py | 230 ++++++++++++++----------- 3 files changed, 137 insertions(+), 113 deletions(-) diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index d270699e58..e9e52fe4a6 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -407,7 +407,6 @@ int mainWindows(int argc, char **argv) { int32_t code = 0; #endif - char *p = taosMemoryCalloc(10, 10); if (global.generateGrant) { dmGenerateGrant(); taosCleanupArgs(); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 707923f200..da8f06895f 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -1077,10 +1077,10 @@ static void taosWriteLog(SLogBuff *pLogBuf) { pLogBuf->writeInterval = 0; } -#define LOG_ROTATE_INTERVAL 1800 +#define LOG_ROTATE_INTERVAL 3600 #ifdef ASSERT_NOT_CORE -#define LOG_INACTIVE_TIME 3600 -#define LOG_ROTATE_BOOT 5 +#define LOG_INACTIVE_TIME 7200 +#define LOG_ROTATE_BOOT 900 #else #define LOG_INACTIVE_TIME 5 #define LOG_ROTATE_BOOT (LOG_INACTIVE_TIME + 1) @@ -1128,9 +1128,12 @@ static void *taosLogRotateFunc(void *param) { continue; } + char fullName[PATH_MAX] = {0}; + snprintf(fullName, sizeof(fullName), "%s%s%s", tsLogDir, TD_DIRSEP, fname); + int64_t mtime = 0; - if ((code = taosStatFile(fname, NULL, &mtime, NULL)) != 0) { - uWarn("%s:%d failed to stat file %s since %s", __func__, __LINE__, fname, tstrerror(code)); + if ((code = taosStatFile(fullName, NULL, &mtime, NULL)) != 0) { + uWarn("%s:%d failed to stat file %s since %s", __func__, __LINE__, fullName, tstrerror(code)); continue; } @@ -1140,9 +1143,6 @@ static void *taosLogRotateFunc(void *param) { continue; } - char fullName[PATH_MAX] = {0}; - snprintf(fullName, sizeof(fullName), "%s%s%s", tsLogDir, TD_DIRSEP, fname); - int32_t days = inactiveSec / 86400 + 1; if (tsLogKeepDays > 0 && days > tsLogKeepDays) { TAOS_UNUSED(taosRemoveFile(fullName)); @@ -1152,7 +1152,7 @@ static void *taosLogRotateFunc(void *param) { } } if ((code = taosCloseDir(&pDir)) != 0) { - uWarn("%s:%d failed to close dir:%s since %s\n", __func__, __LINE__, tsLogDir, tstrerror(code)); + uWarn("%s:%d failed to close dir %s since %s\n", __func__, __LINE__, tsLogDir, tstrerror(code)); } if (tsLogKeepDays > 0) { @@ -1207,6 +1207,7 @@ static void *taosAsyncOutputLog(void *param) { (void)taosThreadAttrInit(&attr); (void)taosThreadAttrSetDetachState(&attr, PTHREAD_CREATE_DETACHED); if (taosThreadCreate(&thread, &attr, taosLogRotateFunc, tsLogObj.logHandle) == 0) { + uInfo("process log rotation"); lastCheckSec = curSec; } else { uWarn("failed to create thread to process log rotation"); diff --git a/tests/system-test/0-others/taosdlog.py b/tests/system-test/0-others/taosdlog.py index a0981d7a72..6349faf587 100644 --- a/tests/system-test/0-others/taosdlog.py +++ b/tests/system-test/0-others/taosdlog.py @@ -1,19 +1,15 @@ -import taos -import sys -import time +import concurrent.futures import os import os.path import platform -import concurrent.futures -import shutil # Add this import +import shutil +import subprocess +import time from util.log import * from util.sql import * from util.cases import * from util.dnodes import * -import time -import socket -import subprocess class TDTestCase: @@ -64,86 +60,44 @@ class TDTestCase: else: tdLog.info("taos.cfg found in %s" % cfgPath) - def checkProcessPid(self,processName): - i=0 - while i<60: - tdLog.info(f"wait stop {processName}") - processPid = subprocess.getstatusoutput(f'ps aux|grep {processName} |grep -v "grep"|awk \'{{print $2}}\'')[1] - tdLog.info(f"times:{i},{processName}-pid:{processPid}") - if(processPid == ""): - break - i += 1 - time.sleep(1) + def closeBin(self, binName): + tdLog.info("Closing %s" % binName) + if platform.system().lower() == 'windows': + psCmd = "for /f %%a in ('wmic process where \"name='%s.exe'\" get processId ^| xargs echo ^| awk ^'{print $2}^' ^&^& echo aa') do @(ps | grep %%a | awk '{print $1}' | xargs)" % binName else: - tdLog.info(f'this processName is not stoped in 60s') + psCmd = "ps -ef | grep -w %s | grep -v grep | awk '{print $2}'" % binName + tdLog.info(f"psCmd:{psCmd}") + + try: + rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + processID = rem.stdout.decode().strip() + tdLog.info(f"processID:{processID}") + except Exception as e: + tdLog.info(f"closeBin error:{e}") + processID = "" + onlyKillOnceWindows = 0 + while(processID): + if not platform.system().lower() == 'windows' or (onlyKillOnceWindows == 0 and platform.system().lower() == 'windows'): + killCmd = "kill -9 %s > /dev/null 2>&1" % processID + if platform.system().lower() == 'windows': + killCmd = "kill -9 %s > nul 2>&1" % processID + tdLog.info(f"killCmd:{killCmd}") + os.system(killCmd) + tdLog.info(f"killed {binName} process {processID}") + onlyKillOnceWindows = 1 + time.sleep(1) + try: + rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + processID = rem.stdout.decode().strip() + except Exception as e: + tdLog.info(f"closeBin error:{e}") + processID = "" def closeTaosd(self, signal=9): - tdLog.info("Closing taosd") - if platform.system().lower() == 'windows': - psCmd = "for /f %%a in ('wmic process where \"name='taosd.exe' and CommandLine like '%%dnode%d%%'\" get processId ^| xargs echo ^| awk ^'{print $2}^' ^&^& echo aa') do @(ps | grep %%a | awk '{print $1}' | xargs)" % (self.index) - else: - psCmd = "ps -ef | grep -w taosd | grep -v grep | awk '{print $2}' | xargs" - tdLog.info(f"psCmd:{psCmd}") - - try: - rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - processID = rem.stdout.decode().strip() - tdLog.info(f"processID:{processID}") - except Exception as e: - tdLog.info(f"closeTaosd error:{e}") - processID = "" - tdLog.info(f"processID:{processID}") - onlyKillOnceWindows = 0 - while(processID): - if not platform.system().lower() == 'windows' or (onlyKillOnceWindows == 0 and platform.system().lower() == 'windows'): - killCmd = "kill -%d %s > /dev/null 2>&1" % (signal, processID) - if platform.system().lower() == 'windows': - killCmd = "kill -%d %s > nul 2>&1" % (signal, processID) - tdLog.info(f"killCmd:{killCmd}") - os.system(killCmd) - tdLog.info(f"killed taosd process {processID}") - onlyKillOnceWindows = 1 - time.sleep(1) - try: - rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - processID = rem.stdout.decode().strip() - except Exception as e: - tdLog.info(f"closeTaosd error:{e}") - processID = "" + self.closeBin("taosd") def closeTaos(self, signal=9): - tdLog.info("Closing taos") - if platform.system().lower() == 'windows': - psCmd = "for /f %%a in ('wmic process where \"name='taos.exe'\" get processId ^| xargs echo ^| awk ^'{print $2}^' ^&^& echo aa') do @(ps | grep %%a | awk '{print $1}' | xargs)" - else: - psCmd = "ps -ef | grep -w taos | grep -v grep | awk '{print $2}'" - tdLog.info(f"psCmd:{psCmd}") - - try: - rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - processID = rem.stdout.decode().strip() - tdLog.info(f"processID:{processID}") - except Exception as e: - tdLog.info(f"closeTaos error:{e}") - processID = "" - tdLog.info(f"processID:{processID}") - onlyKillOnceWindows = 0 - while(processID): - if not platform.system().lower() == 'windows' or (onlyKillOnceWindows == 0 and platform.system().lower() == 'windows'): - killCmd = "kill -%d %s > /dev/null 2>&1" % (signal, processID) - if platform.system().lower() == 'windows': - killCmd = "kill -%d %s > nul 2>&1" % (signal, processID) - tdLog.info(f"killCmd:{killCmd}") - os.system(killCmd) - tdLog.info(f"killed taos process {processID}") - onlyKillOnceWindows = 1 - time.sleep(1) - try: - rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - processID = rem.stdout.decode().strip() - except Exception as e: - tdLog.info(f"closeTaos error:{e}") - processID = "" + self.closeBin("taos") def openBin(self, binPath, waitSec=5): tdLog.info(f"Opening {binPath}") @@ -165,26 +119,30 @@ class TDTestCase: self.openBin(f'{self.taosPath} {args}', waitSec) - def prepare_logoutput(self, desc, port, logOutput): + def prepare_logoutput(self, desc, port, logOutput, skipOpenBin=False): tdLog.info("Preparing %s, port:%s" % (desc, port)) dnodePath = self.buildPath + "/../sim/dnode%s" % port tdLog.info("remove dnodePath:%s" % dnodePath) try: - shutil.rmtree(dnodePath) + if os.path.exists(dnodePath): + shutil.rmtree(dnodePath) except Exception as e: - tdLog.info(f"Failed to remove directory {dnodePath}: {e}") + raise Exception(f"Failed to remove directory {dnodePath}: {e}") try: self.prepareCfg(dnodePath, {"serverPort": port, "dataDir": dnodePath + os.sep + "data", "logDir": dnodePath + os.sep + "log"}) except Exception as e: - tdLog.info(f"Failed to prepare configuration for {dnodePath}: {e}") - return + raise Exception(f"Failed to prepare configuration for {dnodePath}: {e}") try: self.openTaosd(f"-c {dnodePath} {logOutput}") self.openTaos(f"-c {dnodePath} {logOutput}") except Exception as e: - tdLog.info(f"Failed to open taosd or taos for {dnodePath}: {e}") + if(skipOpenBin): + tdLog.info(f"Failed to prepare taosd and taos with log output {logOutput}: {e}") + else: + raise Exception(f"Failed to prepare taosd and taos with log output {logOutput}: {e}") + def prepare_stdout(self): list = self.prepare_list[0] @@ -215,18 +173,18 @@ class TDTestCase: dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[2][1] tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0")) tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0")) - + def prepare_fullpath(self): list = self.prepare_list[3] dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[3][1] self.prepare_logoutput(list[0], list[1], "-o " + dnodePath + "/log0/" ) - + def check_fullpath(self): tdLog.info("Running check fullpath") logPath = self.buildPath + "/../sim/dnode%s/log0/" % self.prepare_list[3][1] tdSql.checkEqual(True, os.path.exists(f"{logPath}taosdlog.0")) tdSql.checkEqual(True, os.path.exists(f"{logPath}taoslog0.0")) - + def prepare_fullname(self): list = self.prepare_list[4] dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[4][1] @@ -238,17 +196,17 @@ class TDTestCase: logPath = self.buildPath + "/../sim/dnode%s/log0/" % self.prepare_list[4][1] tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[4][0] + ".0")) tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[4][0] + "0.0")) - + def prepare_relativepath(self): list = self.prepare_list[5] self.prepare_logoutput(list[0], list[1], "--log-output=" + "log0/") - + def check_relativepath(self): tdLog.info("Running check relativepath") logPath = self.buildPath + "/../sim/dnode%s/log/log0/" % self.prepare_list[5][1] tdSql.checkEqual(True, os.path.exists(logPath + "taosdlog.0")) tdSql.checkEqual(True, os.path.exists(logPath + "taoslog0.0")) - + def prepare_relativename(self): list = self.prepare_list[6] self.prepare_logoutput(list[0], list[1], "-o " + "log0/" + list[0]) @@ -257,7 +215,7 @@ class TDTestCase: logPath = self.buildPath + "/../sim/dnode%s/log/log0/" % self.prepare_list[6][1] tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[6][0] + ".0")) tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[6][0] + "0.0")) - + def prepare_filename(self): list = self.prepare_list[7] self.prepare_logoutput(list[0], list[1], "--log-output " + list[0]) @@ -266,20 +224,29 @@ class TDTestCase: logPath = self.buildPath + "/../sim/dnode%s/log/" % self.prepare_list[7][1] tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[7][0] + ".0")) tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[7][0] + "0.0")) - + def prepare_empty(self): list = self.prepare_list[8] - self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0]) + self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0], True) def check_empty(self): tdLog.info("Running check empty") logPath = self.buildPath + "/../sim/dnode%s/log" % self.prepare_list[8][1] tdSql.checkEqual(False, os.path.exists(f"{logPath}/taosdlog.0")) tdSql.checkEqual(False, os.path.exists(f"{logPath}/taoslog.0")) + def prepare_illegal(self): + list = self.prepare_list[9] + self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0], True) + def check_illegal(self): + tdLog.info("Running check empty") + logPath = self.buildPath + "/../sim/dnode%s/log" % self.prepare_list[9][1] + tdSql.checkEqual(False, os.path.exists(f"{logPath}/taosdlog.0")) + tdSql.checkEqual(False, os.path.exists(f"{logPath}/taoslog.0")) + def prepareCheckResources(self): - self.prepare_list = [["stdout", "10030"], ["stderr", "10031"], ["/dev/null", "10032"], - ["fullpath", "10033"], ["fullname", "10034"], ["relativepath", "10035"], - ["relativename", "10036"], ["filename", "10037"], ["empty", "10038"]] + self.prepare_list = [["stdout", "10030"], ["stderr", "10031"], ["/dev/null", "10032"], ["fullpath", "10033"], + ["fullname", "10034"], ["relativepath", "10035"], ["relativename", "10036"], ["filename", "10037"], + ["empty", "10038"], ["illeg?al", "10039"]] self.check_functions = { self.prepare_stdout: self.check_stdout, self.prepare_stderr: self.check_stderr, @@ -290,6 +257,7 @@ class TDTestCase: self.prepare_relativename: self.check_relativename, self.prepare_filename: self.check_filename, self.prepare_empty: self.check_empty, + self.prepare_illegal: self.check_illegal, } def checkLogOutput(self): @@ -310,14 +278,70 @@ class TDTestCase: future.result() except Exception as e: raise Exception(f"Error in prepare function: {e}") + + def checkLogRotate(self): + tdLog.info("Running check log rotate") + dnodePath = self.buildPath + "/../sim/dnode10050" + logRotateAfterBoot = 6 # LOG_ROTATE_BOOT self.closeTaosd() self.closeTaos() + try: + if os.path.exists(dnodePath): + shutil.rmtree(dnodePath) + self.prepareCfg(dnodePath, {"serverPort": 10050, + "dataDir": dnodePath + os.sep + "data", + "logDir": dnodePath + os.sep + "log", + "logKeepDays": "3" }) + except Exception as e: + raise Exception(f"Failed to prepare configuration for {dnodePath}: {e}") - def is_windows(self): - return os.name == 'nt' + nowSec = int(time.time()) + stubFile99 = f"{dnodePath}/log/taosdlog.99" + stubFile101 = f"{dnodePath}/log/taosdlog.101" + stubGzFile98 = f"{dnodePath}/log/taosdlog.98.gz" + stubGzFile102 = f"{dnodePath}/log/taosdlog.102.gz" + stubFileNow = f"{dnodePath}/log/taosdlog.{nowSec}" + stubGzFileNow = f"{dnodePath}/log/taosdlog.%d.gz" % (nowSec - 1) + stubGzFileKeep = f"{dnodePath}/log/taosdlog.%d.gz" % (nowSec - 86400 * 2) + stubGzFileDel = f"{dnodePath}/log/taosdlog.%d.gz" % (nowSec - 86400 * 3) + stubFiles = [stubFile99, stubFile101, stubGzFile98, stubGzFile102, stubFileNow, stubGzFileNow, stubGzFileKeep, stubGzFileDel] + + try: + os.makedirs(f"{dnodePath}/log", exist_ok=True) + for stubFile in stubFiles: + with open(stubFile, "w") as f: + f.write("test log rotate") + except Exception as e: + raise Exception(f"Failed to prepare log files for {dnodePath}: {e}") + + tdSql.checkEqual(True, os.path.exists(stubFile101)) + tdSql.checkEqual(True, os.path.exists(stubGzFile102)) + tdSql.checkEqual(True, os.path.exists(stubFileNow)) + tdSql.checkEqual(True, os.path.exists(stubGzFileDel)) + + self.openTaosd(f"-c {dnodePath}") + self.openTaos(f"-c {dnodePath}") + + tdLog.info("wait %d seconds for log rotate" % (logRotateAfterBoot + 2)) + time.sleep(logRotateAfterBoot + 2) + + tdSql.checkEqual(True, os.path.exists(stubFile99)) + tdSql.checkEqual(False, os.path.exists(stubFile101)) + tdSql.checkEqual(False, os.path.exists(f'{stubFile101}.gz')) + tdSql.checkEqual(True, os.path.exists(stubGzFile98)) + tdSql.checkEqual(True, os.path.exists(f'{stubFileNow}.gz')) + tdSql.checkEqual(True, os.path.exists(stubGzFileNow)) + tdSql.checkEqual(True, os.path.exists(stubGzFileKeep)) + tdSql.checkEqual(False, os.path.exists(stubGzFile102)) + tdSql.checkEqual(False, os.path.exists(stubGzFileDel)) + tdSql.checkEqual(True, os.path.exists(f"{dnodePath}/log/taosdlog.0")) + tdSql.checkEqual(True, os.path.exists(f"{dnodePath}/log/taoslog0.0")) def run(self): self.checkLogOutput() + self.checkLogRotate() + self.closeTaosd() + self.closeTaos() def stop(self): tdSql.close() From 4c5f46b30885fd46f3345539c3900d073ee9e038 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 3 Dec 2024 11:20:56 +0800 Subject: [PATCH 13/21] test: test case for log output and rotate --- cmake/cmake.options | 4 ---- tests/system-test/0-others/taosdlog.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/cmake/cmake.options b/cmake/cmake.options index 323132ea76..e3b5782d85 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -166,10 +166,6 @@ IF(${BUILD_WITH_ANALYSIS}) set(BUILD_WITH_S3 ON) ENDIF() -IF(${BUILD_TEST}) -add_definitions(-DBUILD_TEST) -ENDIF() - IF(${BUILD_S3}) IF(${BUILD_WITH_S3}) diff --git a/tests/system-test/0-others/taosdlog.py b/tests/system-test/0-others/taosdlog.py index 6349faf587..02f66050be 100644 --- a/tests/system-test/0-others/taosdlog.py +++ b/tests/system-test/0-others/taosdlog.py @@ -115,7 +115,7 @@ class TDTestCase: def openTaosd(self, args="", waitSec=8): self.openBin(f'{self.taosdPath} {args}', waitSec) - def openTaos(self, args="", waitSec=3): + def openTaos(self, args="", waitSec=5): self.openBin(f'{self.taosPath} {args}', waitSec) From 4f33875f16477f9322d280ff3f5622ea1a993724 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 3 Dec 2024 12:25:24 +0800 Subject: [PATCH 14/21] chore: disable log output on windows platform --- source/dnode/mgmt/exe/dmMain.c | 4 ++++ tools/shell/src/shellArguments.c | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index e9e52fe4a6..95674a99c4 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -240,6 +240,7 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { } } else if (strcmp(argv[i], "-k") == 0) { global.generateGrant = true; +#ifndef WINDOWS } else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--log-output") == 0 || strncmp(argv[i], "--log-output=", 13) == 0) { if ((i < argc - 1) || ((i == argc - 1) && strncmp(argv[i], "--log-output=", 13) == 0)) { @@ -264,6 +265,7 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { printf("'%s' requires a parameter\n", argv[i]); return TSDB_CODE_INVALID_CFG; } +#endif } else if (strcmp(argv[i], "-y") == 0) { global.generateCode = true; if (i < argc - 1) { @@ -341,7 +343,9 @@ static void dmPrintHelp() { printf("%s%s%s%s\n", indent, "-e,", indent, DM_ENV_CMD); printf("%s%s%s%s\n", indent, "-E,", indent, DM_ENV_FILE); printf("%s%s%s%s\n", indent, "-k,", indent, DM_MACHINE_CODE); +#ifndef WINDOWS printf("%s%s%s%s\n", indent, "-o, --log-output=OUTPUT", indent, DM_LOG_OUTPUT); +#endif printf("%s%s%s%s\n", indent, "-y,", indent, DM_SET_ENCRYPTKEY); printf("%s%s%s%s\n", indent, "-dm,", indent, DM_MEM_DBG); printf("%s%s%s%s\n", indent, "-V,", indent, DM_VERSION); diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 44686cc5f9..4d1aaf0411 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -79,7 +79,9 @@ void shellPrintHelp() { printf("%s%s%s%s\r\n", indent, "-l,", indent, SHELL_PKT_LEN); printf("%s%s%s%s\r\n", indent, "-n,", indent, SHELL_NET_ROLE); printf("%s%s%s%s\r\n", indent, "-N,", indent, SHELL_PKT_NUM); +#ifndef WINDOWS printf("%s%s%s%s\r\n", indent, "-o,", indent, SHELL_LOG_OUTPUT); +#endif printf("%s%s%s%s\r\n", indent, "-p,", indent, SHELL_PASSWORD); printf("%s%s%s%s\r\n", indent, "-P,", indent, SHELL_PORT); printf("%s%s%s%s\r\n", indent, "-r,", indent, SHELL_RAW_TIME); @@ -140,7 +142,9 @@ static struct argp_option shellOptions[] = { #endif {"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM}, {"bimode", 'B', 0, 0, SHELL_BI_MODE}, +#ifndef WINDOWS {"log-output", 'o', "OUTPUT", 0, SHELL_LOG_OUTPUT}, +#endif {0}, }; @@ -229,6 +233,7 @@ static int32_t shellParseSingleOpt(int32_t key, char *arg) { case 'N': pArgs->pktNum = atoi(arg); break; +#ifndef WINDOWS case 'o': if (strlen(arg) >= PATH_MAX) { printf("failed to set log output since length overflow, max length is %d\n", PATH_MAX); @@ -244,6 +249,7 @@ static int32_t shellParseSingleOpt(int32_t key, char *arg) { return terrno; } break; +#endif #ifdef WEBSOCKET case 'R': pArgs->restful = true; From c92b01309aa35890a7209ac341cb3cab034855e7 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 3 Dec 2024 14:26:33 +0800 Subject: [PATCH 15/21] chore: scheduler CI server --- Jenkinsfile2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 249a8d1c9d..dc94999058 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -505,7 +505,7 @@ pipeline { } } stage('linux test') { - agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "} + agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 "} options { skipDefaultCheckout() } when { changeRequest() From c8df537dbfa876d1bc698f922ca4ec458e9ce343 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 3 Dec 2024 14:52:35 +0800 Subject: [PATCH 16/21] log: enable log output only on linux --- source/dnode/mgmt/exe/dmMain.c | 4 ++-- tools/shell/src/shellArguments.c | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 95674a99c4..4aa76d9284 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -240,7 +240,7 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { } } else if (strcmp(argv[i], "-k") == 0) { global.generateGrant = true; -#ifndef WINDOWS +#if defined(LINUX) } else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--log-output") == 0 || strncmp(argv[i], "--log-output=", 13) == 0) { if ((i < argc - 1) || ((i == argc - 1) && strncmp(argv[i], "--log-output=", 13) == 0)) { @@ -343,7 +343,7 @@ static void dmPrintHelp() { printf("%s%s%s%s\n", indent, "-e,", indent, DM_ENV_CMD); printf("%s%s%s%s\n", indent, "-E,", indent, DM_ENV_FILE); printf("%s%s%s%s\n", indent, "-k,", indent, DM_MACHINE_CODE); -#ifndef WINDOWS +#if defined(LINUX) printf("%s%s%s%s\n", indent, "-o, --log-output=OUTPUT", indent, DM_LOG_OUTPUT); #endif printf("%s%s%s%s\n", indent, "-y,", indent, DM_SET_ENCRYPTKEY); diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 4d1aaf0411..88918a14ac 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -79,7 +79,7 @@ void shellPrintHelp() { printf("%s%s%s%s\r\n", indent, "-l,", indent, SHELL_PKT_LEN); printf("%s%s%s%s\r\n", indent, "-n,", indent, SHELL_NET_ROLE); printf("%s%s%s%s\r\n", indent, "-N,", indent, SHELL_PKT_NUM); -#ifndef WINDOWS +#if defined(LINUX) printf("%s%s%s%s\r\n", indent, "-o,", indent, SHELL_LOG_OUTPUT); #endif printf("%s%s%s%s\r\n", indent, "-p,", indent, SHELL_PASSWORD); @@ -142,7 +142,7 @@ static struct argp_option shellOptions[] = { #endif {"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM}, {"bimode", 'B', 0, 0, SHELL_BI_MODE}, -#ifndef WINDOWS +#if defined(LINUX) {"log-output", 'o', "OUTPUT", 0, SHELL_LOG_OUTPUT}, #endif {0}, @@ -233,7 +233,7 @@ static int32_t shellParseSingleOpt(int32_t key, char *arg) { case 'N': pArgs->pktNum = atoi(arg); break; -#ifndef WINDOWS +#if defined(LINUX) case 'o': if (strlen(arg) >= PATH_MAX) { printf("failed to set log output since length overflow, max length is %d\n", PATH_MAX); From 608ae99579cce5b5fdae46f74bdbd12689fde344 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 3 Dec 2024 15:45:10 +0800 Subject: [PATCH 17/21] test: run taosdlog.py in non asan mode --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 2c525d4396..3e21894a72 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -353,7 +353,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/telemetry.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/backquote_check.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdMonitor.py -,,n,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdlog.py +,,n,system-test,python3 ./test.py -f 0-others/taosdlog.py ,,n,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 ,,n,system-test,python3 ./test.py -f 0-others/udfTest.py ,,n,system-test,python3 ./test.py -f 0-others/udf_create.py From 4a9d72f54d8d2f12c0d0f76cccdbab26ad72a59e Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 3 Dec 2024 16:13:38 +0800 Subject: [PATCH 18/21] log: log rotate logic for community --- source/util/src/tlog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index da8f06895f..5433ed2eed 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -1078,7 +1078,7 @@ static void taosWriteLog(SLogBuff *pLogBuf) { } #define LOG_ROTATE_INTERVAL 3600 -#ifdef ASSERT_NOT_CORE +#if !defined(TD_ENTERPRISE) || defined(ASSERT_NOT_CORE) #define LOG_INACTIVE_TIME 7200 #define LOG_ROTATE_BOOT 900 #else From a26d5f5c8f5b46e0358f019126204983beb46f57 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 3 Dec 2024 17:15:28 +0800 Subject: [PATCH 19/21] test: disable /dev/null to make CI pass --- tests/system-test/0-others/taosdlog.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/system-test/0-others/taosdlog.py b/tests/system-test/0-others/taosdlog.py index 02f66050be..06ac4ce5e4 100644 --- a/tests/system-test/0-others/taosdlog.py +++ b/tests/system-test/0-others/taosdlog.py @@ -166,13 +166,13 @@ class TDTestCase: def prepare_dev_null(self): list = self.prepare_list[2] - self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0]) + # self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0]) def check_dev_null(self): tdLog.info("Running check /dev/null") - dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[2][1] - tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0")) - tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0")) + # dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[2][1] + # tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0")) + # tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0")) def prepare_fullpath(self): list = self.prepare_list[3] From 6dcfe361fe3ad3479b2704fa77c34b7b5acd467b Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 3 Dec 2024 19:10:05 +0800 Subject: [PATCH 20/21] chore: restore Jenkinsfile2 and minor optimization --- Jenkinsfile2 | 2 +- source/util/src/tlog.c | 2 +- source/util/src/tutil.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index dc94999058..249a8d1c9d 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -505,7 +505,7 @@ pipeline { } } stage('linux test') { - agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 "} + agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "} options { skipDefaultCheckout() } when { changeRequest() diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 5433ed2eed..693935575a 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -1198,7 +1198,7 @@ static void *taosAsyncOutputLog(void *param) { break; } - // process the log rotation every LOG_ROTATE_INTERVAL minutes + // process the log rotation every LOG_ROTATE_INTERVAL int64_t curSec = taosGetTimestampMs() / 1000; if (curSec >= lastCheckSec) { if ((curSec - lastCheckSec) >= LOG_ROTATE_INTERVAL) { diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index 306b15909d..dd3c6ece40 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -543,7 +543,7 @@ bool tIsValidFileName(const char *fileName, const char *pattern) { } bool tIsValidFilePath(const char *filePath, const char *pattern) { - const char *filePathPattern = "^[a-zA-Z0-9/\\_.-]+$"; + const char *filePathPattern = "^[a-zA-Z0-9:/\\_.-]+$"; return tIsValidFileName(filePath, pattern ? pattern : filePathPattern); } \ No newline at end of file From 4c0903c4c291d4c3b79aa2d52f29b59bb80891a1 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 4 Dec 2024 08:33:45 +0800 Subject: [PATCH 21/21] enh: boundary check for log output --- source/common/src/tglobal.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 62a1e3f500..76f28d737a 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1019,11 +1019,12 @@ static int32_t taosSetLogOutput(SConfig *pCfg) { if ((pEnd = strrchr(pLog, '/')) || (pEnd = strrchr(pLog, '\\'))) { int32_t pathLen = POINTER_DISTANCE(pEnd, pLog) + 1; if (*pLog == '/' || *pLog == '\\') { - if (pathLen >= PATH_MAX) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + if (pathLen <= 0 || pathLen > PATH_MAX) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); tstrncpy(tsLogDir, pLog, pathLen); } else { int32_t len = strlen(tsLogDir); - if (tsLogDir[len - 1] != '/' && tsLogDir[len - 1] != '\\') { + if (len < 0 || len >= (PATH_MAX - 1)) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + if (len == 0 || (tsLogDir[len - 1] != '/' && tsLogDir[len - 1] != '\\')) { tsLogDir[len++] = TD_DIRSEP_CHAR; } int32_t remain = PATH_MAX - len - 1;