xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

enh: support specify log output by -o option

This commit is contained in:
kailixu 2024-11-29 13:40:33 +08:00
parent 3da2d3d78d
commit b4e6e2dfa8
11 changed files with 199 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/util/tlog.h
View File

@ -72,6 +72,7 @@ extern int32_t simDebugFlag;
extern int32_t tqClientDebugFlag;
int32_t taosInitLogOutput(const char **ppLogName);
int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc);
void taosCloseLog();
void taosResetLog();

2
include/util/tutil.h
View File

@ -56,6 +56,8 @@ void taosIpPort2String(uint32_t ip, uint16_t port, char *str);
void *tmemmem(const char *haystack, int hlen, const char *needle, int nlen);
int32_t parseCfgReal(const char *str, float *out);
bool tIsValidFileName(const char *fileName, const char *pattern);
bool tIsValidFilePath(const char *filePath, const char *pattern);
static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) {
T_MD5_CTX context;

12
source/client/src/clientEnv.c
View File

@ -41,6 +41,10 @@
#include "cus_name.h"
#endif
#ifndef CUS_PROMPT
#define CUS_PROMPT "taos"
#endif
#define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0
@ -954,12 +958,8 @@ void taos_init_imp(void) {
taosHashSetFreeFp(appInfo.pInstMap, destroyAppInst);
deltaToUtcInitOnce();
char logDirName[64] = {0};
#ifdef CUS_PROMPT
snprintf(logDirName, 64, "%slog", CUS_PROMPT);
#else
(void)snprintf(logDirName, 64, "taoslog");
#endif
const char *logDirName = CUS_PROMPT "dlog";
ENV_ERR_RET(taosInitLogOutput(&logDirName), "failed to init log output");
if (taosCreateLog(logDirName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) {
(void)printf(" WARING: Create %s failed:%s. configDir=%s\n", logDirName, strerror(errno), configDir);
tscInitRes = -1;

25
source/common/src/tglobal.c
View File

@ -17,6 +17,7 @@
#include "tglobal.h"
#include "defines.h"
#include "os.h"
#include "osString.h"
#include "tconfig.h"
#include "tgrant.h"
#include "tlog.h"
@ -1010,6 +1011,8 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
extern char *tsLogOutput;
static int32_t taosSetClientLogCfg(SConfig *pCfg) {
SConfigItem *pItem = NULL;
@ -1017,6 +1020,26 @@ static int32_t taosSetClientLogCfg(SConfig *pCfg) {
tstrncpy(tsLogDir, pItem->str, PATH_MAX);
TAOS_CHECK_RETURN(taosExpandDir(tsLogDir, tsLogDir, PATH_MAX));
if (tsLogOutput) {
char *pLog = tsLogOutput;
if (strcasecmp(pLog, "stdout") && strcasecmp(pLog, "stderr") && strcasecmp(pLog, "/dev/null")) {
char *pEnd = NULL;
if ((pEnd = strrchr(pLog, '/')) || (pEnd = strrchr(pLog, '\\'))) {
int32_t pathLen = POINTER_DISTANCE(pEnd, pLog) + 1;
if (*pLog == '/' || *pLog == '\\') {
tstrncpy(tsLogDir, pLog, TMIN(pathLen, PATH_MAX));
} else {
int32_t len = strlen(tsLogDir);
if (tsLogDir[len - 1] != '/' && tsLogDir[len - 1] != '\\') {
tsLogDir[len++] = TD_DIRSEP_CHAR;
}
tstrncpy(tsLogDir + len, pLog, TMIN(PATH_MAX - len - 1, pathLen));
}
cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true);
}
}
}
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "minimalLogDirGB");
tsLogSpace.reserved = (int64_t)(((double)pItem->fval) * 1024 * 1024 * 1024);
@ -1845,6 +1868,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
TAOS_CHECK_GOTO(taosAddSystemCfg(tsCfg), &lino, _exit);
#if 1 // duplicate operation since already loaded in taosCreateLog
if ((code = taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) {
(void)printf("failed to load cfg since %s\n", tstrerror(code));
cfgCleanup(tsCfg);
@ -1858,6 +1882,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
tsCfg = NULL;
TAOS_RETURN(code);
}
#endif
if (tsc) {
TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit);

29
source/dnode/mgmt/exe/dmMain.c
View File

@ -49,6 +49,7 @@
#define DM_ENV_CMD "The env cmd variable string to use when configuring the server, such as: -e 'TAOS_FQDN=td1'."
#define DM_ENV_FILE "The env variable file path to use when configuring the server, default is './.env', .env text can be 'TAOS_FQDN=td1'."
#define DM_MACHINE_CODE "Get machine code."
#define DM_LOG_OUTPUT "Specify log output. Options:\n\r\t\t\t stdout, stderr, /dev/null, <directory>, <directory>/<filename>, <filename>\n\r\t\t\t * If OUTPUT contains an absolute directory, logs will be stored in that directory instead of logDir.\n\r\t\t\t * If OUTPUT contains a relative directory, logs will be stored in the directory combined with logDir and the relative directory."
#define DM_VERSION "Print program version."
#define DM_EMAIL "<support@taosdata.com>"
#define DM_MEM_DBG "Enable memory debug"
@ -185,6 +186,7 @@ static void dmSetSignalHandle() {
}
extern bool generateNewMeta;
extern char *tsLogOutput;
static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
global.startTime = taosGetTimestampMs();
@ -239,6 +241,25 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
}
} else if (strcmp(argv[i], "-k") == 0) {
global.generateGrant = true;
} else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--log-output") == 0) {
if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) {
printf("failed to set log output since length overflow, max length is %d\n", PATH_MAX);
return TSDB_CODE_INVALID_CFG;
}
tsLogOutput = taosMemoryMalloc(PATH_MAX);
if (!tsLogOutput) {
printf("failed to set log output: '%s' since %s\n", argv[i], tstrerror(terrno));
return terrno;
}
if (taosExpandDir(argv[i], tsLogOutput, PATH_MAX) != 0) {
printf("failed to expand log output: '%s' since %s\n", argv[i], tstrerror(terrno));
return terrno;
}
} else {
printf("'-o' requires a parameter\n");
return TSDB_CODE_INVALID_CFG;
}
} else if (strcmp(argv[i], "-y") == 0) {
global.generateCode = true;
if (i < argc - 1) {
@ -316,6 +337,7 @@ static void dmPrintHelp() {
printf("%s%s%s%s\n", indent, "-e,", indent, DM_ENV_CMD);
printf("%s%s%s%s\n", indent, "-E,", indent, DM_ENV_FILE);
printf("%s%s%s%s\n", indent, "-k,", indent, DM_MACHINE_CODE);
printf("%s%s%s%s\n", indent, "-o, --log-output=OUTPUT", indent, DM_LOG_OUTPUT);
printf("%s%s%s%s\n", indent, "-y,", indent, DM_SET_ENCRYPTKEY);
printf("%s%s%s%s\n", indent, "-dm,", indent, DM_MEM_DBG);
printf("%s%s%s%s\n", indent, "-V,", indent, DM_VERSION);
@ -340,8 +362,11 @@ static int32_t dmCheckS3() {
}
static int32_t dmInitLog() {
return taosCreateLog(CUS_PROMPT "dlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs,
0);
const char *logName = CUS_PROMPT "dlog";
TAOS_CHECK_RETURN(taosInitLogOutput(&logName));
return taosCreateLog(logName, 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0);
}
static void taosCleanupArgs() {

4
source/libs/function/src/udfd.c
View File

@ -1564,8 +1564,8 @@ static void udfdPrintVersion() {
}
static int32_t udfdInitLog() {
char logName[12] = {0};
snprintf(logName, sizeof(logName), "%slog", "udfd");
const char *logName = "udfdlog";
TAOS_CHECK_RETURN(taosInitLogOutput(&logName));
return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, NULL, 0);
}

4
source/os/src/osDir.c
View File

@ -339,7 +339,7 @@ int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) {
OS_PARAM_CHECK(dirname);
OS_PARAM_CHECK(outname);
wordexp_t full_path;
int32_t code = wordexp(dirname, &full_path, 0);
int32_t code = wordexp(dirname, &full_path, 0);
switch (code) {
case 0:
break;
@ -347,7 +347,7 @@ int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) {
wordfree(&full_path);
// FALL THROUGH
default:
return code;
return terrno = TSDB_CODE_INVALID_PARA;
}
if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) {

2
source/os/src/osEnv.c
View File

@ -89,7 +89,7 @@ int32_t osDefaultInit() {
}
tstrncpy(tsDataDir, TD_DATA_DIR_PATH, sizeof(tsDataDir));
tstrncpy(tsLogDir, TD_LOG_DIR_PATH, sizeof(tsLogDir));
if (strlen(tsTempDir) == 0){
if (strlen(tsTempDir) == 0) {
tstrncpy(tsTempDir, TD_TMP_DIR_PATH, sizeof(tsTempDir));
}

85
source/util/src/tlog.c
View File

@ -51,6 +51,13 @@
#define LOG_EDITION_FLG ("C")
#endif
typedef enum {
LOG_OUTPUT_FILE = 0, // default
LOG_OUTPUT_STDOUT = 1, // stdout set by -o option on the command line
LOG_OUTPUT_STDERR = 2, // stderr set by -e option on the command line
LOG_OUTPUT_NULL = 4, // /dev/null set by -o option on the command line
} ELogOutputType;
typedef struct {
char *buffer;
int32_t buffStart;
@ -73,6 +80,7 @@ typedef struct {
int32_t openInProgress;
int64_t lastKeepFileSec;
int64_t timestampToday;
int8_t outputType; // ELogOutputType
pid_t pid;
char logName[PATH_MAX];
char slowLogName[PATH_MAX];
@ -96,6 +104,8 @@ bool tsAssert = true;
#endif
int32_t tsNumOfLogLines = 10000000;
int32_t tsLogKeepDays = 0;
char *tsLogOutput = NULL;
LogFp tsLogFp = NULL;
int64_t tsNumOfErrorLogs = 0;
int64_t tsNumOfInfoLogs = 0;
@ -234,6 +244,58 @@ int32_t taosInitSlowLog() {
return 0;
}
int32_t taosInitLogOutput(const char **ppLogName) {
const char *pLog = tsLogOutput;
const char *pLogName = NULL;
if (pLog) {
if (!tIsValidFilePath(pLog, NULL)) {
fprintf(stderr, "invalid log output destination:%s, contains illegal char\n", pLog);
return TSDB_CODE_INVALID_CFG;
}
if (0 == strcasecmp(pLog, "stdout")) {
tsLogObj.outputType = LOG_OUTPUT_STDOUT;
if (ppLogName) *ppLogName = pLog;
return 0;
}
if (0 == strcasecmp(pLog, "stderr")) {
tsLogObj.outputType = LOG_OUTPUT_STDERR;
if (ppLogName) *ppLogName = pLog;
return 0;
}
if (0 == strcasecmp(pLog, "/dev/null")) {
tsLogObj.outputType = LOG_OUTPUT_NULL;
if (ppLogName) *ppLogName = pLog;
return 0;
}
int32_t len = strlen(pLog);
if (len < 1) {
fprintf(stderr, "invalid log output destination:%s, should not be empty\n", pLog);
return TSDB_CODE_INVALID_CFG;
}
const char *p = pLog + (len - 1);
if (*p == '/' || *p == '\\') {
return 0;
}
if ((p = strrchr(pLog, '/')) || (p = strrchr(pLog, '\\'))) {
pLogName = p + 1;
} else {
pLogName = pLog;
}
if (strcmp(pLogName, ".") == 0 || strcmp(pLogName, "..") == 0) {
fprintf(stderr, "invalid log output destination:%s\n", pLog);
return TSDB_CODE_INVALID_CFG;
}
if (!tIsValidFileName(pLogName, NULL)) {
fprintf(stderr, "invalid log output destination:%s, contains illegal char\n", pLog);
return TSDB_CODE_INVALID_CFG;
}
if (ppLogName) *ppLogName = pLogName;
}
return 0;
}
int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) {
if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0;
int32_t code = osUpdate();
@ -241,6 +303,11 @@ int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) {
uError("failed to update os info, reason:%s", tstrerror(code));
}
if (tsLogObj.outputType == LOG_OUTPUT_STDOUT || tsLogObj.outputType == LOG_OUTPUT_STDERR ||
tsLogObj.outputType == LOG_OUTPUT_NULL) {
return 0;
}
TAOS_CHECK_RETURN(taosInitNormalLog(logName, maxFiles));
if (tsc) {
TAOS_CHECK_RETURN(taosInitSlowLog());
@ -283,6 +350,7 @@ void taosCloseLog() {
taosMemoryFreeClear(tsLogObj.logHandle);
tsLogObj.logHandle = NULL;
}
taosMemoryFreeClear(tsLogOutput);
}
static bool taosLockLogFile(TdFilePtr pFile) {
@ -374,7 +442,7 @@ static OldFileKeeper *taosOpenNewFile() {
}
TdFilePtr pOldFile = tsLogObj.logHandle->pFile;
atomic_store_ptr(&tsLogObj.logHandle->pFile, pFile);
tsLogObj.logHandle->pFile = pFile;
tsLogObj.lines = 0;
tsLogObj.openInProgress = 0;
OldFileKeeper *oldFileKeeper = taosMemoryMalloc(sizeof(OldFileKeeper));
@ -666,7 +734,7 @@ static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *b
if (tsAsyncLog) {
TAOS_UNUSED(taosPushLogBuffer(tsLogObj.logHandle, buffer, len));
} else {
TAOS_UNUSED(taosWriteFile(atomic_load_ptr(tsLogObj.logHandle->pFile), buffer, len));
TAOS_UNUSED(taosWriteFile(tsLogObj.logHandle->pFile, buffer, len));
}
if (tsNumOfLogLines > 0) {
@ -680,10 +748,19 @@ static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *b
}
}
if (dflag & DEBUG_SCREEN) {
int fd = 0;
if (tsLogObj.outputType == LOG_OUTPUT_FILE) {
if (dflag & DEBUG_SCREEN) fd = 1;
} else if (tsLogObj.outputType == LOG_OUTPUT_STDOUT) {
fd = 1;
} else if (tsLogObj.outputType == LOG_OUTPUT_STDERR) {
fd = 2;
}
if (fd) {
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
if (write(1, buffer, (uint32_t)len) < 0) {
if (write(fd, buffer, (uint32_t)len) < 0) {
TAOS_UNUSED(printf("failed to write log to screen, reason:%s\n", strerror(errno)));
}
#pragma GCC diagnostic pop

27
source/util/src/tutil.c
View File

@ -16,6 +16,7 @@
#define _DEFAULT_SOURCE
#include "tutil.h"
#include "tlog.h"
#include "regex.h"
void *tmemmem(const char *haystack, int32_t hlen, const char *needle, int32_t nlen) {
const char *limit;
@ -520,3 +521,29 @@ int32_t parseCfgReal(const char *str, float *out) {
*out = val;
return TSDB_CODE_SUCCESS;
}
bool tIsValidFileName(const char *fileName, const char *pattern) {
const char *fileNamePattern = "^[a-zA-Z0-9_.-]+$";
regex_t fileNameReg;
if (pattern) fileNamePattern = pattern;
if (regcomp(&fileNameReg, fileNamePattern, REG_EXTENDED) != 0) {
fprintf(stderr, "failed to compile file name pattern:%s\n", fileNamePattern);
return false;
}
int32_t code = regexec(&fileNameReg, fileName, 0, NULL, 0);
regfree(&fileNameReg);
if (code != 0) {
return false;
}
return true;
}
bool tIsValidFilePath(const char *filePath, const char *pattern) {
const char *filePathPattern = "^[a-zA-Z0-9/\\_.-]+$";
return tIsValidFileName(filePath, pattern ? pattern : filePathPattern);
}

26
tools/shell/src/shellArguments.c
View File

@ -49,7 +49,12 @@
#define SHELL_PKT_LEN "Packet length used for net test, default is 1024 bytes."
#define SHELL_PKT_NUM "Packet numbers used for net test, default is 100."
#define SHELL_BI_MODE "Set BI mode"
#define SHELL_VERSION "Print program version."
#define SHELL_LOG_OUTPUT \
"Specify log output. Options:\n\r\t\t\t stdout, stderr, /dev/null, <directory>, <directory>/<filename>, " \
"<filename>\n\r\t\t\t * If OUTPUT contains an absolute directory, logs will be stored in that directory " \
"instead of logDir.\n\r\t\t\t * If OUTPUT contains a relative directory, logs will be stored in the directory " \
"combined with logDir and the relative directory."
#define SHELL_VERSION "Print program version."
#ifdef WEBSOCKET
#define SHELL_DSN "Use dsn to connect to the cloud server or to a remote server which provides WebSocket connection."
@ -74,6 +79,7 @@ void shellPrintHelp() {
printf("%s%s%s%s\r\n", indent, "-l,", indent, SHELL_PKT_LEN);
printf("%s%s%s%s\r\n", indent, "-n,", indent, SHELL_NET_ROLE);
printf("%s%s%s%s\r\n", indent, "-N,", indent, SHELL_PKT_NUM);
printf("%s%s%s%s\r\n", indent, "-o,", indent, SHELL_LOG_OUTPUT);
printf("%s%s%s%s\r\n", indent, "-p,", indent, SHELL_PASSWORD);
printf("%s%s%s%s\r\n", indent, "-P,", indent, SHELL_PORT);
printf("%s%s%s%s\r\n", indent, "-r,", indent, SHELL_RAW_TIME);
@ -134,6 +140,7 @@ static struct argp_option shellOptions[] = {
#endif
{"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM},
{"bimode", 'B', 0, 0, SHELL_BI_MODE},
{"log-output", 'o', "OUTPUT", 0, SHELL_LOG_OUTPUT},
{0},
};
@ -152,6 +159,8 @@ static void shellParseArgsUseArgp(int argc, char *argv[]) {
#define ARGP_ERR_UNKNOWN E2BIG
#endif
extern char* tsLogOutput;
static int32_t shellParseSingleOpt(int32_t key, char *arg) {
SShellArgs *pArgs = &shell.args;
@ -222,6 +231,21 @@ static int32_t shellParseSingleOpt(int32_t key, char *arg) {
case 'N':
pArgs->pktNum = atoi(arg);
break;
case 'o':
if (strlen(arg) >= PATH_MAX) {
printf("failed to set log output since length overflow, max length is %d\n", PATH_MAX);
return TSDB_CODE_INVALID_CFG;
}
tsLogOutput = taosMemoryMalloc(PATH_MAX);
if (!tsLogOutput) {
printf("failed to set log output: '%s' since %s\n", arg, tstrerror(terrno));
return terrno;
}
if (taosExpandDir(arg, tsLogOutput, PATH_MAX) != 0) {
printf("failed to expand log output: '%s' since %s\n", arg, tstrerror(terrno));
return terrno;
}
break;
#ifdef WEBSOCKET
case 'R':
pArgs->restful = true;