diff --git a/include/common/cos_cp.h b/include/common/cos_cp.h index 29532c0265..e54941f5d3 100644 --- a/include/common/cos_cp.h +++ b/include/common/cos_cp.h @@ -44,7 +44,7 @@ typedef struct { char file_path[TSDB_FILENAME_LEN]; // local file path int64_t file_size; // local file size, for upload - int32_t file_last_modified; // local file last modified time, for upload + int64_t file_last_modified; // local file last modified time, for upload char file_md5[64]; // md5 of the local file content, for upload, reserved char object_name[128]; // object name @@ -67,9 +67,9 @@ int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint); int32_t cos_cp_dump(SCheckpoint* checkpoint); void cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPart* parts, int64_t* consume_bytes); void cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag, uint64_t crc64); -void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int32_t mtime, +void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int64_t mtime, char const* upload_id, int64_t part_size); -bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int32_t mtime); +bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int64_t mtime); void cos_cp_build_download(SCheckpoint* checkpoint, char const* filepath, char const* object_name, int64_t object_size, char const* object_lmtime, char const* object_etag, int64_t part_size); diff --git a/include/os/osFile.h b/include/os/osFile.h index 536dee268a..1c397f3042 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -82,9 +82,9 @@ int32_t taosUnLockFile(TdFilePtr pFile); int32_t taosUmaskFile(int32_t maskVal); -int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *atime); +int32_t taosStatFile(const char *path, int64_t *size, int64_t *mtime, int64_t *atime); int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno); -int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime); +int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int64_t *mtime); bool taosCheckExistFile(const char *pathname); int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence); diff --git a/include/util/tlog.h b/include/util/tlog.h index a6c87593d1..78ca6ffe15 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -41,6 +41,7 @@ extern bool tsAsyncLog; extern bool tsAssert; extern int32_t tsNumOfLogLines; extern int32_t tsLogKeepDays; +extern char *tsLogOutput; extern LogFp tsLogFp; extern int64_t tsNumOfErrorLogs; extern int64_t tsNumOfInfoLogs; @@ -71,7 +72,7 @@ extern int32_t sndDebugFlag; extern int32_t simDebugFlag; extern int32_t tqClientDebugFlag; - +int32_t taosInitLogOutput(const char **ppLogName); int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc); void taosCloseLog(); void taosResetLog(); diff --git a/include/util/tutil.h b/include/util/tutil.h index aa3b774e84..c1b6b2505b 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -56,6 +56,8 @@ void taosIpPort2String(uint32_t ip, uint16_t port, char *str); void *tmemmem(const char *haystack, int hlen, const char *needle, int nlen); int32_t parseCfgReal(const char *str, float *out); +bool tIsValidFileName(const char *fileName, const char *pattern); +bool tIsValidFilePath(const char *filePath, const char *pattern); static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) { T_MD5_CTX context; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index b0be3a4d3b..072cbee2f4 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -41,6 +41,10 @@ #include "cus_name.h" #endif +#ifndef CUS_PROMPT +#define CUS_PROMPT "tao" +#endif + #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -954,14 +958,10 @@ void taos_init_imp(void) { taosHashSetFreeFp(appInfo.pInstMap, destroyAppInst); deltaToUtcInitOnce(); - char logDirName[64] = {0}; -#ifdef CUS_PROMPT - snprintf(logDirName, 64, "%slog", CUS_PROMPT); -#else - (void)snprintf(logDirName, 64, "taoslog"); -#endif - if (taosCreateLog(logDirName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { - (void)printf(" WARING: Create %s failed:%s. configDir=%s\n", logDirName, strerror(errno), configDir); + const char *logName = CUS_PROMPT "slog"; + ENV_ERR_RET(taosInitLogOutput(&logName), "failed to init log output"); + if (taosCreateLog(logName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { + (void)printf(" WARING: Create %s failed:%s. configDir=%s\n", logName, strerror(errno), configDir); tscInitRes = -1; return; } diff --git a/source/common/src/cos.c b/source/common/src/cos.c index a7e69ddc4c..de4f65f2ad 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -775,7 +775,7 @@ _exit: TAOS_RETURN(code); } -static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const char *file, int32_t lmtime, +static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const char *file, int64_t lmtime, char const *object_name, int64_t contentLength, S3PutProperties *put_prop, put_object_callback_data *data) { int32_t code = 0, lino = 0; @@ -963,7 +963,7 @@ _exit: int32_t s3PutObjectFromFile2ByEp(const char *file, const char *object_name, int8_t withcp, int8_t epIndex) { int32_t code = 0; - int32_t lmtime = 0; + int64_t lmtime = 0; const char *filename = 0; uint64_t contentLength = 0; const char *cacheControl = 0, *contentType = 0, *md5 = 0; @@ -1040,7 +1040,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t w static int32_t s3PutObjectFromFileOffsetByEp(const char *file, const char *object_name, int64_t offset, int64_t size, int8_t epIndex) { int32_t code = 0; - int32_t lmtime = 0; + int64_t lmtime = 0; const char *filename = 0; uint64_t contentLength = 0; const char *cacheControl = 0, *contentType = 0, *md5 = 0; @@ -1847,7 +1847,7 @@ _exit: typedef struct { int64_t size; - int32_t atime; + int64_t atime; char name[TSDB_FILENAME_LEN]; } SEvictFile; diff --git a/source/common/src/cos_cp.c b/source/common/src/cos_cp.c index 078b14c9e8..3469e8ecec 100644 --- a/source/common/src/cos_cp.c +++ b/source/common/src/cos_cp.c @@ -350,7 +350,7 @@ void cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag checkpoint->parts[part_index].crc64 = crc64; } -void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int32_t mtime, +void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int64_t mtime, char const* upload_id, int64_t part_size) { int i = 0; @@ -375,7 +375,7 @@ void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t static bool cos_cp_verify_md5(SCheckpoint* cp) { return true; } -bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int32_t mtime) { +bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int64_t mtime) { if (cos_cp_verify_md5(checkpoint) && checkpoint->file_size == size && checkpoint->file_last_modified == mtime) { return true; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 46026b586b..5076006f29 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -17,6 +17,7 @@ #include "tglobal.h" #include "defines.h" #include "os.h" +#include "osString.h" #include "tconfig.h" #include "tgrant.h" #include "tlog.h" @@ -1010,12 +1011,43 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { TAOS_RETURN(TSDB_CODE_SUCCESS); } +static int32_t taosSetLogOutput(SConfig *pCfg) { + if (tsLogOutput) { + char *pLog = tsLogOutput; + char *pEnd = NULL; + if (strcasecmp(pLog, "stdout") && strcasecmp(pLog, "stderr") && strcasecmp(pLog, "/dev/null")) { + if ((pEnd = strrchr(pLog, '/')) || (pEnd = strrchr(pLog, '\\'))) { + int32_t pathLen = POINTER_DISTANCE(pEnd, pLog) + 1; + if (*pLog == '/' || *pLog == '\\') { + if (pathLen <= 0 || pathLen > PATH_MAX) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + tstrncpy(tsLogDir, pLog, pathLen); + } else { + int32_t len = strlen(tsLogDir); + if (len < 0 || len >= (PATH_MAX - 1)) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + if (len == 0 || (tsLogDir[len - 1] != '/' && tsLogDir[len - 1] != '\\')) { + tsLogDir[len++] = TD_DIRSEP_CHAR; + } + int32_t remain = PATH_MAX - len - 1; + if (remain < pathLen) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + tstrncpy(tsLogDir + len, pLog, pathLen); + } + TAOS_CHECK_RETURN(cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true)); + } + } else { + tstrncpy(tsLogDir, pLog, PATH_MAX); + TAOS_CHECK_RETURN(cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true)); + } + } + return 0; +} + static int32_t taosSetClientLogCfg(SConfig *pCfg) { SConfigItem *pItem = NULL; TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "logDir"); tstrncpy(tsLogDir, pItem->str, PATH_MAX); TAOS_CHECK_RETURN(taosExpandDir(tsLogDir, tsLogDir, PATH_MAX)); + TAOS_CHECK_RETURN(taosSetLogOutput(pCfg)); TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "minimalLogDirGB"); tsLogSpace.reserved = (int64_t)(((double)pItem->fval) * 1024 * 1024 * 1024); @@ -1860,8 +1892,10 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile } if (tsc) { + TAOS_CHECK_GOTO(taosSetClientLogCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit); } else { + TAOS_CHECK_GOTO(taosSetClientLogCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosUpdateServerCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosSetServerCfg(tsCfg), &lino, _exit); diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index ade5e16894..4aa76d9284 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -49,6 +49,7 @@ #define DM_ENV_CMD "The env cmd variable string to use when configuring the server, such as: -e 'TAOS_FQDN=td1'." #define DM_ENV_FILE "The env variable file path to use when configuring the server, default is './.env', .env text can be 'TAOS_FQDN=td1'." #define DM_MACHINE_CODE "Get machine code." +#define DM_LOG_OUTPUT "Specify log output. Options:\n\r\t\t\t stdout, stderr, /dev/null, , /, \n\r\t\t\t * If OUTPUT contains an absolute directory, logs will be stored in that directory instead of logDir.\n\r\t\t\t * If OUTPUT contains a relative directory, logs will be stored in the directory combined with logDir and the relative directory." #define DM_VERSION "Print program version." #define DM_EMAIL "" #define DM_MEM_DBG "Enable memory debug" @@ -239,6 +240,32 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { } } else if (strcmp(argv[i], "-k") == 0) { global.generateGrant = true; +#if defined(LINUX) + } else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--log-output") == 0 || + strncmp(argv[i], "--log-output=", 13) == 0) { + if ((i < argc - 1) || ((i == argc - 1) && strncmp(argv[i], "--log-output=", 13) == 0)) { + int32_t klen = strlen(argv[i]); + int32_t vlen = klen < 13 ? strlen(argv[++i]) : klen - 13; + const char *val = argv[i]; + if (klen >= 13) val += 13; + if (vlen <= 0 || vlen >= PATH_MAX) { + printf("failed to set log output since invalid vlen:%d, valid range: [1, %d)\n", vlen, PATH_MAX); + return TSDB_CODE_INVALID_CFG; + } + tsLogOutput = taosMemoryMalloc(PATH_MAX); + if (!tsLogOutput) { + printf("failed to set log output: '%s' since %s\n", val, tstrerror(terrno)); + return terrno; + } + if (taosExpandDir(val, tsLogOutput, PATH_MAX) != 0) { + printf("failed to expand log output: '%s' since %s\n", val, tstrerror(terrno)); + return terrno; + } + } else { + printf("'%s' requires a parameter\n", argv[i]); + return TSDB_CODE_INVALID_CFG; + } +#endif } else if (strcmp(argv[i], "-y") == 0) { global.generateCode = true; if (i < argc - 1) { @@ -316,6 +343,9 @@ static void dmPrintHelp() { printf("%s%s%s%s\n", indent, "-e,", indent, DM_ENV_CMD); printf("%s%s%s%s\n", indent, "-E,", indent, DM_ENV_FILE); printf("%s%s%s%s\n", indent, "-k,", indent, DM_MACHINE_CODE); +#if defined(LINUX) + printf("%s%s%s%s\n", indent, "-o, --log-output=OUTPUT", indent, DM_LOG_OUTPUT); +#endif printf("%s%s%s%s\n", indent, "-y,", indent, DM_SET_ENCRYPTKEY); printf("%s%s%s%s\n", indent, "-dm,", indent, DM_MEM_DBG); printf("%s%s%s%s\n", indent, "-V,", indent, DM_VERSION); @@ -340,8 +370,11 @@ static int32_t dmCheckS3() { } static int32_t dmInitLog() { - return taosCreateLog(CUS_PROMPT "dlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, - 0); + const char *logName = CUS_PROMPT "dlog"; + + TAOS_CHECK_RETURN(taosInitLogOutput(&logName)); + + return taosCreateLog(logName, 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0); } static void taosCleanupArgs() { diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 74ec310f70..7859ee4c66 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -660,7 +660,7 @@ static int32_t tsdbDoS3Migrate(SRTNer *rtner) { int32_t lcn = fobj->f->lcn; if (/*lcn < 1 && */ taosCheckExistFile(fobj->fname)) { - int32_t mtime = 0; + int64_t mtime = 0; int64_t size = 0; int32_t r = taosStatFile(fobj->fname, &size, &mtime, NULL); if (size > chunksize && mtime < rtner->now - tsS3UploadDelaySec) { @@ -687,7 +687,7 @@ static int32_t tsdbDoS3Migrate(SRTNer *rtner) { tsdbTFileLastChunkName(rtner->tsdb, fobj->f, fname1); if (taosCheckExistFile(fname1)) { - int32_t mtime = 0; + int64_t mtime = 0; int64_t size = 0; if (taosStatFile(fname1, &size, &mtime, NULL) != 0) { tsdbError("vgId:%d, %s failed at %s:%d ", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, __LINE__); diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 0ee14f7820..bbfd43d5f7 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -1564,8 +1564,8 @@ static void udfdPrintVersion() { } static int32_t udfdInitLog() { - char logName[12] = {0}; - snprintf(logName, sizeof(logName), "%slog", "udfd"); + const char *logName = "udfdlog"; + TAOS_CHECK_RETURN(taosInitLogOutput(&logName)); return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, NULL, 0); } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index b40a9eeefe..78f13a58ab 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -326,7 +326,7 @@ static int32_t walRepairLogFileTs(SWal* pWal, bool* updateMeta) { } walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); - int32_t mtime = 0; + int64_t mtime = 0; if (taosStatFile(fnameStr, NULL, &mtime, NULL) < 0) { wError("vgId:%d, failed to stat file due to %s, file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 5db60df1ec..25245d2df9 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -303,7 +303,7 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue; char filename[1024]; - (void)snprintf(filename, sizeof(filename), "%s/%s", dirname, taosGetDirEntryName(de)); + (void)snprintf(filename, sizeof(filename), "%s%s%s", dirname, TD_DIRSEP, taosGetDirEntryName(de)); if (taosDirEntryIsDir(de)) { continue; } else { @@ -349,7 +349,7 @@ int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) { wordfree(&full_path); // FALL THROUGH default: - return code; + return terrno = TSDB_CODE_INVALID_PARA; } if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) { diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 41b34a9030..832deb574d 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -89,7 +89,7 @@ int32_t osDefaultInit() { } tstrncpy(tsDataDir, TD_DATA_DIR_PATH, sizeof(tsDataDir)); tstrncpy(tsLogDir, TD_LOG_DIR_PATH, sizeof(tsLogDir)); - if (strlen(tsTempDir) == 0){ + if (strlen(tsTempDir) == 0) { tstrncpy(tsTempDir, TD_TMP_DIR_PATH, sizeof(tsTempDir)); } diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index a3747535f6..5030eb1f67 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -273,7 +273,7 @@ int32_t taosRenameFile(const char *oldName, const char *newName) { #endif } -int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *atime) { +int32_t taosStatFile(const char *path, int64_t *size, int64_t *mtime, int64_t *atime) { OS_PARAM_CHECK(path); #ifdef WINDOWS struct _stati64 fileStat; @@ -544,7 +544,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { return liOffset.QuadPart; } -int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { +int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int64_t *mtime) { if (pFile == NULL || pFile->hFile == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; @@ -571,7 +571,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { ULARGE_INTEGER ull; ull.LowPart = lastWriteTime.dwLowDateTime; ull.HighPart = lastWriteTime.dwHighDateTime; - *mtime = (int32_t)((ull.QuadPart - 116444736000000000ULL) / 10000000ULL); + *mtime = (int64_t)((ull.QuadPart - 116444736000000000ULL) / 10000000ULL); } return 0; } @@ -937,7 +937,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { return ret; } -int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { +int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int64_t *mtime) { if (pFile == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index f70b145dbc..693935575a 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -51,6 +51,13 @@ #define LOG_EDITION_FLG ("C") #endif +typedef enum { + LOG_OUTPUT_FILE = 0, // default + LOG_OUTPUT_STDOUT = 1, // stdout set by -o option on the command line + LOG_OUTPUT_STDERR = 2, // stderr set by -o option on the command line + LOG_OUTPUT_NULL = 4, // /dev/null set by -o option on the command line +} ELogOutputType; + typedef struct { char *buffer; int32_t buffStart; @@ -73,6 +80,7 @@ typedef struct { int32_t openInProgress; int64_t lastKeepFileSec; int64_t timestampToday; + int8_t outputType; // ELogOutputType pid_t pid; char logName[PATH_MAX]; char slowLogName[PATH_MAX]; @@ -86,6 +94,7 @@ static int8_t tsLogInited = 0; static SLogObj tsLogObj = {.fileNum = 1, .slowHandle = NULL}; static int64_t tsAsyncLogLostLines = 0; static int32_t tsDaylightActive; /* Currently in daylight saving time. */ +static SRWLatch tsLogRotateLatch = 0; bool tsLogEmbedded = 0; bool tsAsyncLog = true; @@ -96,6 +105,7 @@ bool tsAssert = true; #endif int32_t tsNumOfLogLines = 10000000; int32_t tsLogKeepDays = 0; +char *tsLogOutput = NULL; LogFp tsLogFp = NULL; int64_t tsNumOfErrorLogs = 0; int64_t tsNumOfInfoLogs = 0; @@ -234,6 +244,58 @@ int32_t taosInitSlowLog() { return 0; } +int32_t taosInitLogOutput(const char **ppLogName) { + const char *pLog = tsLogOutput; + const char *pLogName = NULL; + if (pLog) { + if (!tIsValidFilePath(pLog, NULL)) { + fprintf(stderr, "invalid log output destination:%s, contains illegal char\n", pLog); + return TSDB_CODE_INVALID_CFG; + } + if (0 == strcasecmp(pLog, "stdout")) { + tsLogObj.outputType = LOG_OUTPUT_STDOUT; + if (ppLogName) *ppLogName = pLog; + return 0; + } + if (0 == strcasecmp(pLog, "stderr")) { + tsLogObj.outputType = LOG_OUTPUT_STDERR; + if (ppLogName) *ppLogName = pLog; + return 0; + } + if (0 == strcasecmp(pLog, "/dev/null")) { + tsLogObj.outputType = LOG_OUTPUT_NULL; + if (ppLogName) *ppLogName = pLog; + return 0; + } + int32_t len = strlen(pLog); + if (len < 1) { + fprintf(stderr, "invalid log output destination:%s, should not be empty\n", pLog); + return TSDB_CODE_INVALID_CFG; + } + const char *p = pLog + (len - 1); + if (*p == '/' || *p == '\\') { + return 0; + } + + if ((p = strrchr(pLog, '/')) || (p = strrchr(pLog, '\\'))) { + pLogName = p + 1; + } else { + pLogName = pLog; + } + if (strcmp(pLogName, ".") == 0 || strcmp(pLogName, "..") == 0) { + fprintf(stderr, "invalid log output destination:%s\n", pLog); + return TSDB_CODE_INVALID_CFG; + } + + if (!tIsValidFileName(pLogName, NULL)) { + fprintf(stderr, "invalid log output destination:%s, contains illegal char\n", pLog); + return TSDB_CODE_INVALID_CFG; + } + if (ppLogName) *ppLogName = pLogName; + } + return 0; +} + int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) { if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0; int32_t code = osUpdate(); @@ -241,6 +303,11 @@ int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) { uError("failed to update os info, reason:%s", tstrerror(code)); } + if (tsLogObj.outputType == LOG_OUTPUT_STDOUT || tsLogObj.outputType == LOG_OUTPUT_STDERR || + tsLogObj.outputType == LOG_OUTPUT_NULL) { + return 0; + } + TAOS_CHECK_RETURN(taosInitNormalLog(logName, maxFiles)); if (tsc) { TAOS_CHECK_RETURN(taosInitSlowLog()); @@ -283,6 +350,7 @@ void taosCloseLog() { taosMemoryFreeClear(tsLogObj.logHandle); tsLogObj.logHandle = NULL; } + taosMemoryFreeClear(tsLogOutput); } static bool taosLockLogFile(TdFilePtr pFile) { @@ -340,10 +408,6 @@ static void taosKeepOldLog(char *oldName) { } } } - - if (tsLogKeepDays > 0) { - taosRemoveOldFiles(tsLogDir, tsLogKeepDays); - } } typedef struct { TdFilePtr pOldFile; @@ -392,11 +456,16 @@ static OldFileKeeper *taosOpenNewFile() { static void *taosThreadToCloseOldFile(void *param) { if (!param) return NULL; + taosWLockLatch(&tsLogRotateLatch); OldFileKeeper *oldFileKeeper = (OldFileKeeper *)param; taosSsleep(20); taosCloseLogByFd(oldFileKeeper->pOldFile); taosKeepOldLog(oldFileKeeper->keepName); taosMemoryFree(oldFileKeeper); + if (tsLogKeepDays > 0) { + taosRemoveOldFiles(tsLogDir, tsLogKeepDays); + } + taosWUnLockLatch(&tsLogRotateLatch); return NULL; } @@ -532,8 +601,8 @@ static void decideLogFileName(const char *fn, int32_t maxFileNum) { static void decideLogFileNameFlag() { char name[PATH_MAX + 50] = "\0"; - int32_t logstat0_mtime = 0; - int32_t logstat1_mtime = 0; + int64_t logstat0_mtime = 0; + int64_t logstat1_mtime = 0; bool log0Exist = false; bool log1Exist = false; @@ -680,10 +749,19 @@ static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *b } } - if (dflag & DEBUG_SCREEN) { + int fd = 0; + if (tsLogObj.outputType == LOG_OUTPUT_FILE) { + if (dflag & DEBUG_SCREEN) fd = 1; + } else if (tsLogObj.outputType == LOG_OUTPUT_STDOUT) { + fd = 1; + } else if (tsLogObj.outputType == LOG_OUTPUT_STDERR) { + fd = 2; + } + + if (fd) { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-result" - if (write(1, buffer, (uint32_t)len) < 0) { + if (write(fd, buffer, (uint32_t)len) < 0) { TAOS_UNUSED(printf("failed to write log to screen, reason:%s\n", strerror(errno))); } #pragma GCC diagnostic pop @@ -999,6 +1077,92 @@ static void taosWriteLog(SLogBuff *pLogBuf) { pLogBuf->writeInterval = 0; } +#define LOG_ROTATE_INTERVAL 3600 +#if !defined(TD_ENTERPRISE) || defined(ASSERT_NOT_CORE) +#define LOG_INACTIVE_TIME 7200 +#define LOG_ROTATE_BOOT 900 +#else +#define LOG_INACTIVE_TIME 5 +#define LOG_ROTATE_BOOT (LOG_INACTIVE_TIME + 1) +#endif + +static void *taosLogRotateFunc(void *param) { + setThreadName("logRotate"); + int32_t code = 0; + taosWLockLatch(&tsLogRotateLatch); + // compress or remove the old log files + TdDirPtr pDir = taosOpenDir(tsLogDir); + if (!pDir) goto _exit; + TdDirEntryPtr de = NULL; + while ((de = taosReadDir(pDir))) { + if (taosDirEntryIsDir(de)) { + continue; + } + char *fname = taosGetDirEntryName(de); + if (!fname) { + continue; + } + + char *pSec = strrchr(fname, '.'); + if (!pSec) { + continue; + } + char *pIter = pSec; + bool isSec = true; + while (*(++pIter)) { + if (!isdigit(*pIter)) { + isSec = false; + break; + } + } + if (!isSec) { + continue; + } + + int64_t fileSec = 0; + if ((code = taosStr2int64(pSec + 1, &fileSec)) != 0) { + uWarn("%s:%d failed to convert %s to int64 since %s", __func__, __LINE__, pSec + 1, tstrerror(code)); + continue; + } + if (fileSec <= 100) { + continue; + } + + char fullName[PATH_MAX] = {0}; + snprintf(fullName, sizeof(fullName), "%s%s%s", tsLogDir, TD_DIRSEP, fname); + + int64_t mtime = 0; + if ((code = taosStatFile(fullName, NULL, &mtime, NULL)) != 0) { + uWarn("%s:%d failed to stat file %s since %s", __func__, __LINE__, fullName, tstrerror(code)); + continue; + } + + int64_t inactiveSec = taosGetTimestampMs() / 1000 - mtime; + + if (inactiveSec < LOG_INACTIVE_TIME) { + continue; + } + + int32_t days = inactiveSec / 86400 + 1; + if (tsLogKeepDays > 0 && days > tsLogKeepDays) { + TAOS_UNUSED(taosRemoveFile(fullName)); + uInfo("file:%s is removed, days:%d, keepDays:%d, sed:%" PRId64, fullName, days, tsLogKeepDays, fileSec); + } else { + taosKeepOldLog(fullName); // compress + } + } + if ((code = taosCloseDir(&pDir)) != 0) { + uWarn("%s:%d failed to close dir %s since %s\n", __func__, __LINE__, tsLogDir, tstrerror(code)); + } + + if (tsLogKeepDays > 0) { + taosRemoveOldFiles(tsLogDir, tsLogKeepDays); + } +_exit: + taosWUnLockLatch(&tsLogRotateLatch); + return NULL; +} + static void *taosAsyncOutputLog(void *param) { SLogBuff *pLogBuf = (SLogBuff *)tsLogObj.logHandle; SLogBuff *pSlowBuf = (SLogBuff *)tsLogObj.slowHandle; @@ -1007,6 +1171,7 @@ static void *taosAsyncOutputLog(void *param) { int32_t count = 0; int32_t updateCron = 0; int32_t writeInterval = 0; + int64_t lastCheckSec = taosGetTimestampMs() / 1000 - (LOG_ROTATE_INTERVAL - LOG_ROTATE_BOOT); while (1) { if (pSlowBuf) { @@ -1032,6 +1197,26 @@ static void *taosAsyncOutputLog(void *param) { if (pSlowBuf) taosWriteSlowLog(pSlowBuf); break; } + + // process the log rotation every LOG_ROTATE_INTERVAL + int64_t curSec = taosGetTimestampMs() / 1000; + if (curSec >= lastCheckSec) { + if ((curSec - lastCheckSec) >= LOG_ROTATE_INTERVAL) { + TdThread thread; + TdThreadAttr attr; + (void)taosThreadAttrInit(&attr); + (void)taosThreadAttrSetDetachState(&attr, PTHREAD_CREATE_DETACHED); + if (taosThreadCreate(&thread, &attr, taosLogRotateFunc, tsLogObj.logHandle) == 0) { + uInfo("process log rotation"); + lastCheckSec = curSec; + } else { + uWarn("failed to create thread to process log rotation"); + } + (void)taosThreadAttrDestroy(&attr); + } + } else if (curSec < lastCheckSec) { + lastCheckSec = curSec; + } } return NULL; diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index 48338e7996..dd3c6ece40 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "tutil.h" #include "tlog.h" +#include "regex.h" void *tmemmem(const char *haystack, int32_t hlen, const char *needle, int32_t nlen) { const char *limit; @@ -520,3 +521,29 @@ int32_t parseCfgReal(const char *str, float *out) { *out = val; return TSDB_CODE_SUCCESS; } + +bool tIsValidFileName(const char *fileName, const char *pattern) { + const char *fileNamePattern = "^[a-zA-Z0-9_.-]+$"; + + regex_t fileNameReg; + + if (pattern) fileNamePattern = pattern; + + if (regcomp(&fileNameReg, fileNamePattern, REG_EXTENDED) != 0) { + fprintf(stderr, "failed to compile file name pattern:%s\n", fileNamePattern); + return false; + } + + int32_t code = regexec(&fileNameReg, fileName, 0, NULL, 0); + regfree(&fileNameReg); + if (code != 0) { + return false; + } + return true; +} + +bool tIsValidFilePath(const char *filePath, const char *pattern) { + const char *filePathPattern = "^[a-zA-Z0-9:/\\_.-]+$"; + + return tIsValidFileName(filePath, pattern ? pattern : filePathPattern); +} \ No newline at end of file diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 055bd7f9c6..0c05901093 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -354,6 +354,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/telemetry.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/backquote_check.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdMonitor.py +,,n,system-test,python3 ./test.py -f 0-others/taosdlog.py ,,n,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 ,,n,system-test,python3 ./test.py -f 0-others/udfTest.py ,,n,system-test,python3 ./test.py -f 0-others/udf_create.py diff --git a/tests/system-test/0-others/taosdlog.py b/tests/system-test/0-others/taosdlog.py index d4698960cd..06ac4ce5e4 100644 --- a/tests/system-test/0-others/taosdlog.py +++ b/tests/system-test/0-others/taosdlog.py @@ -1,66 +1,351 @@ -import taos -import sys -import time -import os - -from util.log import * -from util.sql import * -from util.cases import * -from util.dnodes import * - -class TDTestCase: - - def init(self, conn, logSql, replicaVar=1): - self.replicaVar = int(replicaVar) - tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor()) - - def getBuildPath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files or "taosd.exe" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root) - len("/build/bin")] - break - return buildPath - - def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring - tdSql.prepare() - # time.sleep(2) - tdSql.query("create user testpy pass 'testpy'") - - buildPath = self.getBuildPath() - if (buildPath == ""): - tdLog.exit("taosd not found!") - else: - tdLog.info("taosd found in %s" % buildPath) - logPath = buildPath + "/../sim/dnode1/log" - tdLog.info("log path: %s" % logPath) - - tdDnodes.stop(1) - time.sleep(2) - tdSql.error("select * from information_schema.ins_databases") - os.system("rm -rf %s" % logPath) - if os.path.exists(logPath) == True: - tdLog.exit("log pat still exist!") - - tdDnodes.start(1) - time.sleep(2) - if os.path.exists(logPath) != True: - tdLog.exit("log pat is not generated!") - - tdSql.query("select * from information_schema.ins_databases") - - def stop(self): - tdSql.close() - tdLog.success(f"{__file__} successfully executed") - -tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) +import concurrent.futures +import os +import os.path +import platform +import shutil +import subprocess +import time + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + tdSql.prepare() + self.buildPath = self.getBuildPath() + if (self.buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % self.buildPath) + self.taosdPath = self.buildPath + "/build/bin/taosd" + self.taosPath = self.buildPath + "/build/bin/taos" + self.logPath = self.buildPath + "/../sim/dnode1/log" + tdLog.info("taosd path: %s" % self.taosdPath) + tdLog.info("taos path: %s" % self.taosPath) + tdLog.info("log path: %s" % self.logPath) + self.commonCfgDict = {} + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def prepareCfg(self, cfgPath, cfgDict): + tdLog.info("make dir %s" % cfgPath) + os.makedirs(cfgPath, exist_ok=True) + with open(cfgPath + "/taos.cfg", "w") as f: + for key in self.commonCfgDict: + f.write("%s %s\n" % (key, self.commonCfgDict[key])) + for key in cfgDict: + f.write("%s %s\n" % (key, cfgDict[key])) + if not os.path.exists(cfgPath + "/taos.cfg"): + tdLog.exit("taos.cfg not found in %s" % cfgPath) + else: + tdLog.info("taos.cfg found in %s" % cfgPath) + + def closeBin(self, binName): + tdLog.info("Closing %s" % binName) + if platform.system().lower() == 'windows': + psCmd = "for /f %%a in ('wmic process where \"name='%s.exe'\" get processId ^| xargs echo ^| awk ^'{print $2}^' ^&^& echo aa') do @(ps | grep %%a | awk '{print $1}' | xargs)" % binName + else: + psCmd = "ps -ef | grep -w %s | grep -v grep | awk '{print $2}'" % binName + tdLog.info(f"psCmd:{psCmd}") + + try: + rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + processID = rem.stdout.decode().strip() + tdLog.info(f"processID:{processID}") + except Exception as e: + tdLog.info(f"closeBin error:{e}") + processID = "" + onlyKillOnceWindows = 0 + while(processID): + if not platform.system().lower() == 'windows' or (onlyKillOnceWindows == 0 and platform.system().lower() == 'windows'): + killCmd = "kill -9 %s > /dev/null 2>&1" % processID + if platform.system().lower() == 'windows': + killCmd = "kill -9 %s > nul 2>&1" % processID + tdLog.info(f"killCmd:{killCmd}") + os.system(killCmd) + tdLog.info(f"killed {binName} process {processID}") + onlyKillOnceWindows = 1 + time.sleep(1) + try: + rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + processID = rem.stdout.decode().strip() + except Exception as e: + tdLog.info(f"closeBin error:{e}") + processID = "" + + def closeTaosd(self, signal=9): + self.closeBin("taosd") + + def closeTaos(self, signal=9): + self.closeBin("taos") + + def openBin(self, binPath, waitSec=5): + tdLog.info(f"Opening {binPath}") + try: + process = subprocess.Popen(binPath, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + time.sleep(waitSec) + if process.poll() is None: + tdLog.info(f"{binPath} started successfully") + else: + error = process.stderr.read().decode(encoding="utf-8").strip() + raise Exception(f"Failed to start {binPath}: {error}") + except Exception as e: + raise Exception(f"Failed to start {binPath}: %s" % repr(e)) + + def openTaosd(self, args="", waitSec=8): + self.openBin(f'{self.taosdPath} {args}', waitSec) + + def openTaos(self, args="", waitSec=5): + self.openBin(f'{self.taosPath} {args}', waitSec) + + + def prepare_logoutput(self, desc, port, logOutput, skipOpenBin=False): + tdLog.info("Preparing %s, port:%s" % (desc, port)) + dnodePath = self.buildPath + "/../sim/dnode%s" % port + tdLog.info("remove dnodePath:%s" % dnodePath) + try: + if os.path.exists(dnodePath): + shutil.rmtree(dnodePath) + except Exception as e: + raise Exception(f"Failed to remove directory {dnodePath}: {e}") + try: + self.prepareCfg(dnodePath, {"serverPort": port, + "dataDir": dnodePath + os.sep + "data", + "logDir": dnodePath + os.sep + "log"}) + except Exception as e: + raise Exception(f"Failed to prepare configuration for {dnodePath}: {e}") + try: + self.openTaosd(f"-c {dnodePath} {logOutput}") + self.openTaos(f"-c {dnodePath} {logOutput}") + except Exception as e: + if(skipOpenBin): + tdLog.info(f"Failed to prepare taosd and taos with log output {logOutput}: {e}") + else: + raise Exception(f"Failed to prepare taosd and taos with log output {logOutput}: {e}") + + + def prepare_stdout(self): + list = self.prepare_list[0] + self.prepare_logoutput(list[0], list[1], "-o " + list[0]) + + def check_stdout(self): + tdLog.info("Running check stdout") + dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[0][1] + tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0")) + tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0")) + + def prepare_stderr(self): + list = self.prepare_list[1] + self.prepare_logoutput(list[0], list[1], "--log-output " + list[0]) + + def check_stderr(self): + tdLog.info("Running check stderr") + dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[1][1] + tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0")) + tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0")) + + def prepare_dev_null(self): + list = self.prepare_list[2] + # self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0]) + + def check_dev_null(self): + tdLog.info("Running check /dev/null") + # dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[2][1] + # tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0")) + # tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0")) + + def prepare_fullpath(self): + list = self.prepare_list[3] + dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[3][1] + self.prepare_logoutput(list[0], list[1], "-o " + dnodePath + "/log0/" ) + + def check_fullpath(self): + tdLog.info("Running check fullpath") + logPath = self.buildPath + "/../sim/dnode%s/log0/" % self.prepare_list[3][1] + tdSql.checkEqual(True, os.path.exists(f"{logPath}taosdlog.0")) + tdSql.checkEqual(True, os.path.exists(f"{logPath}taoslog0.0")) + + def prepare_fullname(self): + list = self.prepare_list[4] + dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[4][1] + self.prepare_logoutput(list[0], list[1], "--log-output " + dnodePath + "/log0/" + list[0]) + dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[4][1] + + def check_fullname(self): + tdLog.info("Running check fullname") + logPath = self.buildPath + "/../sim/dnode%s/log0/" % self.prepare_list[4][1] + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[4][0] + ".0")) + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[4][0] + "0.0")) + + def prepare_relativepath(self): + list = self.prepare_list[5] + self.prepare_logoutput(list[0], list[1], "--log-output=" + "log0/") + + def check_relativepath(self): + tdLog.info("Running check relativepath") + logPath = self.buildPath + "/../sim/dnode%s/log/log0/" % self.prepare_list[5][1] + tdSql.checkEqual(True, os.path.exists(logPath + "taosdlog.0")) + tdSql.checkEqual(True, os.path.exists(logPath + "taoslog0.0")) + + def prepare_relativename(self): + list = self.prepare_list[6] + self.prepare_logoutput(list[0], list[1], "-o " + "log0/" + list[0]) + def check_relativename(self): + tdLog.info("Running check relativename") + logPath = self.buildPath + "/../sim/dnode%s/log/log0/" % self.prepare_list[6][1] + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[6][0] + ".0")) + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[6][0] + "0.0")) + + def prepare_filename(self): + list = self.prepare_list[7] + self.prepare_logoutput(list[0], list[1], "--log-output " + list[0]) + def check_filename(self): + tdLog.info("Running check filename") + logPath = self.buildPath + "/../sim/dnode%s/log/" % self.prepare_list[7][1] + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[7][0] + ".0")) + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[7][0] + "0.0")) + + def prepare_empty(self): + list = self.prepare_list[8] + self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0], True) + def check_empty(self): + tdLog.info("Running check empty") + logPath = self.buildPath + "/../sim/dnode%s/log" % self.prepare_list[8][1] + tdSql.checkEqual(False, os.path.exists(f"{logPath}/taosdlog.0")) + tdSql.checkEqual(False, os.path.exists(f"{logPath}/taoslog.0")) + + def prepare_illegal(self): + list = self.prepare_list[9] + self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0], True) + def check_illegal(self): + tdLog.info("Running check empty") + logPath = self.buildPath + "/../sim/dnode%s/log" % self.prepare_list[9][1] + tdSql.checkEqual(False, os.path.exists(f"{logPath}/taosdlog.0")) + tdSql.checkEqual(False, os.path.exists(f"{logPath}/taoslog.0")) + + def prepareCheckResources(self): + self.prepare_list = [["stdout", "10030"], ["stderr", "10031"], ["/dev/null", "10032"], ["fullpath", "10033"], + ["fullname", "10034"], ["relativepath", "10035"], ["relativename", "10036"], ["filename", "10037"], + ["empty", "10038"], ["illeg?al", "10039"]] + self.check_functions = { + self.prepare_stdout: self.check_stdout, + self.prepare_stderr: self.check_stderr, + self.prepare_dev_null: self.check_dev_null, + self.prepare_fullpath: self.check_fullpath, + self.prepare_fullname: self.check_fullname, + self.prepare_relativepath: self.check_relativepath, + self.prepare_relativename: self.check_relativename, + self.prepare_filename: self.check_filename, + self.prepare_empty: self.check_empty, + self.prepare_illegal: self.check_illegal, + } + + def checkLogOutput(self): + self.closeTaosd() + self.closeTaos() + self.prepareCheckResources() + with concurrent.futures.ThreadPoolExecutor() as executor: + prepare_futures = [executor.submit(prepare_func) for prepare_func, _ in self.check_functions.items()] + for future in concurrent.futures.as_completed(prepare_futures): + try: + future.result() + except Exception as e: + raise Exception(f"Error in prepare function: {e}") + + check_futures = [executor.submit(check_func) for _, check_func in self.check_functions.items()] + for future in concurrent.futures.as_completed(check_futures): + try: + future.result() + except Exception as e: + raise Exception(f"Error in prepare function: {e}") + + def checkLogRotate(self): + tdLog.info("Running check log rotate") + dnodePath = self.buildPath + "/../sim/dnode10050" + logRotateAfterBoot = 6 # LOG_ROTATE_BOOT + self.closeTaosd() + self.closeTaos() + try: + if os.path.exists(dnodePath): + shutil.rmtree(dnodePath) + self.prepareCfg(dnodePath, {"serverPort": 10050, + "dataDir": dnodePath + os.sep + "data", + "logDir": dnodePath + os.sep + "log", + "logKeepDays": "3" }) + except Exception as e: + raise Exception(f"Failed to prepare configuration for {dnodePath}: {e}") + + nowSec = int(time.time()) + stubFile99 = f"{dnodePath}/log/taosdlog.99" + stubFile101 = f"{dnodePath}/log/taosdlog.101" + stubGzFile98 = f"{dnodePath}/log/taosdlog.98.gz" + stubGzFile102 = f"{dnodePath}/log/taosdlog.102.gz" + stubFileNow = f"{dnodePath}/log/taosdlog.{nowSec}" + stubGzFileNow = f"{dnodePath}/log/taosdlog.%d.gz" % (nowSec - 1) + stubGzFileKeep = f"{dnodePath}/log/taosdlog.%d.gz" % (nowSec - 86400 * 2) + stubGzFileDel = f"{dnodePath}/log/taosdlog.%d.gz" % (nowSec - 86400 * 3) + stubFiles = [stubFile99, stubFile101, stubGzFile98, stubGzFile102, stubFileNow, stubGzFileNow, stubGzFileKeep, stubGzFileDel] + + try: + os.makedirs(f"{dnodePath}/log", exist_ok=True) + for stubFile in stubFiles: + with open(stubFile, "w") as f: + f.write("test log rotate") + except Exception as e: + raise Exception(f"Failed to prepare log files for {dnodePath}: {e}") + + tdSql.checkEqual(True, os.path.exists(stubFile101)) + tdSql.checkEqual(True, os.path.exists(stubGzFile102)) + tdSql.checkEqual(True, os.path.exists(stubFileNow)) + tdSql.checkEqual(True, os.path.exists(stubGzFileDel)) + + self.openTaosd(f"-c {dnodePath}") + self.openTaos(f"-c {dnodePath}") + + tdLog.info("wait %d seconds for log rotate" % (logRotateAfterBoot + 2)) + time.sleep(logRotateAfterBoot + 2) + + tdSql.checkEqual(True, os.path.exists(stubFile99)) + tdSql.checkEqual(False, os.path.exists(stubFile101)) + tdSql.checkEqual(False, os.path.exists(f'{stubFile101}.gz')) + tdSql.checkEqual(True, os.path.exists(stubGzFile98)) + tdSql.checkEqual(True, os.path.exists(f'{stubFileNow}.gz')) + tdSql.checkEqual(True, os.path.exists(stubGzFileNow)) + tdSql.checkEqual(True, os.path.exists(stubGzFileKeep)) + tdSql.checkEqual(False, os.path.exists(stubGzFile102)) + tdSql.checkEqual(False, os.path.exists(stubGzFileDel)) + tdSql.checkEqual(True, os.path.exists(f"{dnodePath}/log/taosdlog.0")) + tdSql.checkEqual(True, os.path.exists(f"{dnodePath}/log/taoslog0.0")) + + def run(self): + self.checkLogOutput() + self.checkLogRotate() + self.closeTaosd() + self.closeTaos() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 442329674d..88918a14ac 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -49,7 +49,12 @@ #define SHELL_PKT_LEN "Packet length used for net test, default is 1024 bytes." #define SHELL_PKT_NUM "Packet numbers used for net test, default is 100." #define SHELL_BI_MODE "Set BI mode" -#define SHELL_VERSION "Print program version." +#define SHELL_LOG_OUTPUT \ + "Specify log output. Options:\n\r\t\t\t stdout, stderr, /dev/null, , /, " \ + "\n\r\t\t\t * If OUTPUT contains an absolute directory, logs will be stored in that directory " \ + "instead of logDir.\n\r\t\t\t * If OUTPUT contains a relative directory, logs will be stored in the directory " \ + "combined with logDir and the relative directory." +#define SHELL_VERSION "Print program version." #ifdef WEBSOCKET #define SHELL_DSN "Use dsn to connect to the cloud server or to a remote server which provides WebSocket connection." @@ -74,6 +79,9 @@ void shellPrintHelp() { printf("%s%s%s%s\r\n", indent, "-l,", indent, SHELL_PKT_LEN); printf("%s%s%s%s\r\n", indent, "-n,", indent, SHELL_NET_ROLE); printf("%s%s%s%s\r\n", indent, "-N,", indent, SHELL_PKT_NUM); +#if defined(LINUX) + printf("%s%s%s%s\r\n", indent, "-o,", indent, SHELL_LOG_OUTPUT); +#endif printf("%s%s%s%s\r\n", indent, "-p,", indent, SHELL_PASSWORD); printf("%s%s%s%s\r\n", indent, "-P,", indent, SHELL_PORT); printf("%s%s%s%s\r\n", indent, "-r,", indent, SHELL_RAW_TIME); @@ -134,6 +142,9 @@ static struct argp_option shellOptions[] = { #endif {"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM}, {"bimode", 'B', 0, 0, SHELL_BI_MODE}, +#if defined(LINUX) + {"log-output", 'o', "OUTPUT", 0, SHELL_LOG_OUTPUT}, +#endif {0}, }; @@ -222,6 +233,23 @@ static int32_t shellParseSingleOpt(int32_t key, char *arg) { case 'N': pArgs->pktNum = atoi(arg); break; +#if defined(LINUX) + case 'o': + if (strlen(arg) >= PATH_MAX) { + printf("failed to set log output since length overflow, max length is %d\n", PATH_MAX); + return TSDB_CODE_INVALID_CFG; + } + tsLogOutput = taosMemoryMalloc(PATH_MAX); + if (!tsLogOutput) { + printf("failed to set log output: '%s' since %s\n", arg, tstrerror(terrno)); + return terrno; + } + if (taosExpandDir(arg, tsLogOutput, PATH_MAX) != 0) { + printf("failed to expand log output: '%s' since %s\n", arg, tstrerror(terrno)); + return terrno; + } + break; +#endif #ifdef WEBSOCKET case 'R': pArgs->restful = true;