enh: support specify log output by -o option

include/util/tlog.h

@@ -72,6 +72,7 @@ extern int32_t simDebugFlag;
 
 extern int32_t tqClientDebugFlag;
 
+int32_t taosInitLogOutput(const char **ppLogName);
 int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc);
 void    taosCloseLog();
 void    taosResetLog();

include/util/tutil.h

@@ -56,6 +56,8 @@ void     taosIpPort2String(uint32_t ip, uint16_t port, char *str);
 void *tmemmem(const char *haystack, int hlen, const char *needle, int nlen);
 
 int32_t parseCfgReal(const char *str, float *out);
+bool    tIsValidFileName(const char *fileName, const char *pattern);
+bool    tIsValidFilePath(const char *filePath, const char *pattern);
 
 static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) {
   T_MD5_CTX context;

source/client/src/clientEnv.c

@@ -41,6 +41,10 @@
 #include "cus_name.h"
 #endif
 
+#ifndef CUS_PROMPT
+#define CUS_PROMPT "taos"
+#endif
+
 #define TSC_VAR_NOT_RELEASE 1
 #define TSC_VAR_RELEASED    0
 
@@ -954,12 +958,8 @@ void taos_init_imp(void) {
   taosHashSetFreeFp(appInfo.pInstMap, destroyAppInst);
   deltaToUtcInitOnce();
 
-  char logDirName[64] = {0};
+  const char *logDirName = CUS_PROMPT "dlog";
-#ifdef CUS_PROMPT
+  ENV_ERR_RET(taosInitLogOutput(&logDirName), "failed to init log output");
-  snprintf(logDirName, 64, "%slog", CUS_PROMPT);
-#else
-  (void)snprintf(logDirName, 64, "taoslog");
-#endif
   if (taosCreateLog(logDirName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) {
     (void)printf(" WARING: Create %s failed:%s. configDir=%s\n", logDirName, strerror(errno), configDir);
     tscInitRes = -1;

source/common/src/tglobal.c

@@ -17,6 +17,7 @@
 #include "tglobal.h"
 #include "defines.h"
 #include "os.h"
+#include "osString.h"
 #include "tconfig.h"
 #include "tgrant.h"
 #include "tlog.h"
@@ -1010,6 +1011,8 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
   TAOS_RETURN(TSDB_CODE_SUCCESS);
 }
 
+extern char *tsLogOutput;
+
 static int32_t taosSetClientLogCfg(SConfig *pCfg) {
   SConfigItem *pItem = NULL;
 
@@ -1017,6 +1020,26 @@ static int32_t taosSetClientLogCfg(SConfig *pCfg) {
   tstrncpy(tsLogDir, pItem->str, PATH_MAX);
   TAOS_CHECK_RETURN(taosExpandDir(tsLogDir, tsLogDir, PATH_MAX));
 
+  if (tsLogOutput) {
+    char *pLog = tsLogOutput;
+    if (strcasecmp(pLog, "stdout") && strcasecmp(pLog, "stderr") && strcasecmp(pLog, "/dev/null")) {
+      char *pEnd = NULL;
+      if ((pEnd = strrchr(pLog, '/')) || (pEnd = strrchr(pLog, '\\'))) {
+        int32_t pathLen = POINTER_DISTANCE(pEnd, pLog) + 1;
+        if (*pLog == '/' || *pLog == '\\') {
+          tstrncpy(tsLogDir, pLog, TMIN(pathLen, PATH_MAX));
+        } else {
+          int32_t len = strlen(tsLogDir);
+          if (tsLogDir[len - 1] != '/' && tsLogDir[len - 1] != '\\') {
+            tsLogDir[len++] = TD_DIRSEP_CHAR;
+          }
+          tstrncpy(tsLogDir + len, pLog, TMIN(PATH_MAX - len - 1, pathLen));
+        }
+        cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true);
+      }
+    }
+  }
+
   TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "minimalLogDirGB");
   tsLogSpace.reserved = (int64_t)(((double)pItem->fval) * 1024 * 1024 * 1024);
 
@@ -1845,6 +1868,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
 
   TAOS_CHECK_GOTO(taosAddSystemCfg(tsCfg), &lino, _exit);
 
+#if 1  // duplicate operation since already loaded in taosCreateLog
   if ((code = taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) {
     (void)printf("failed to load cfg since %s\n", tstrerror(code));
     cfgCleanup(tsCfg);
@@ -1858,6 +1882,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
     tsCfg = NULL;
     TAOS_RETURN(code);
   }
+#endif
 
   if (tsc) {
     TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit);

source/dnode/mgmt/exe/dmMain.c

@@ -49,6 +49,7 @@
 #define DM_ENV_CMD       "The env cmd variable string to use when configuring the server, such as: -e 'TAOS_FQDN=td1'."
 #define DM_ENV_FILE      "The env variable file path to use when configuring the server, default is './.env', .env text can be 'TAOS_FQDN=td1'."
 #define DM_MACHINE_CODE  "Get machine code."
+#define DM_LOG_OUTPUT    "Specify log output. Options:\n\r\t\t\t   stdout, stderr, /dev/null, <directory>, <directory>/<filename>, <filename>\n\r\t\t\t   * If OUTPUT contains an absolute directory, logs will be stored in that directory instead of logDir.\n\r\t\t\t   * If OUTPUT contains a relative directory, logs will be stored in the directory combined with logDir and the relative directory."
 #define DM_VERSION       "Print program version."
 #define DM_EMAIL         "<support@taosdata.com>"
 #define DM_MEM_DBG       "Enable memory debug"
@@ -185,6 +186,7 @@ static void dmSetSignalHandle() {
 }
 
 extern bool generateNewMeta;
+extern char *tsLogOutput;
 
 static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
   global.startTime = taosGetTimestampMs();
@@ -239,6 +241,25 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
       }
     } else if (strcmp(argv[i], "-k") == 0) {
       global.generateGrant = true;
+    } else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--log-output") == 0) {
+      if (i < argc - 1) {
+        if (strlen(argv[++i]) >= PATH_MAX) {
+          printf("failed to set log output since length overflow, max length is %d\n", PATH_MAX);
+          return TSDB_CODE_INVALID_CFG;
+        }
+        tsLogOutput = taosMemoryMalloc(PATH_MAX);
+        if (!tsLogOutput) {
+          printf("failed to set log output: '%s' since %s\n", argv[i], tstrerror(terrno));
+          return terrno;
+        }
+        if (taosExpandDir(argv[i], tsLogOutput, PATH_MAX) != 0) {
+          printf("failed to expand log output: '%s' since %s\n", argv[i], tstrerror(terrno));
+          return terrno;
+        }
+      } else {
+        printf("'-o' requires a parameter\n");
+        return TSDB_CODE_INVALID_CFG;
+      }
     } else if (strcmp(argv[i], "-y") == 0) {
       global.generateCode = true;
       if (i < argc - 1) {
@@ -316,6 +337,7 @@ static void dmPrintHelp() {
   printf("%s%s%s%s\n", indent, "-e,", indent, DM_ENV_CMD);
   printf("%s%s%s%s\n", indent, "-E,", indent, DM_ENV_FILE);
   printf("%s%s%s%s\n", indent, "-k,", indent, DM_MACHINE_CODE);
+  printf("%s%s%s%s\n", indent, "-o, --log-output=OUTPUT", indent, DM_LOG_OUTPUT);
   printf("%s%s%s%s\n", indent, "-y,", indent, DM_SET_ENCRYPTKEY);
   printf("%s%s%s%s\n", indent, "-dm,", indent, DM_MEM_DBG);
   printf("%s%s%s%s\n", indent, "-V,", indent, DM_VERSION);
@@ -340,8 +362,11 @@ static int32_t dmCheckS3() {
 }
 
 static int32_t dmInitLog() {
-  return taosCreateLog(CUS_PROMPT "dlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs,
+  const char *logName = CUS_PROMPT "dlog";
-                       0);
+
+  TAOS_CHECK_RETURN(taosInitLogOutput(&logName));
+
+  return taosCreateLog(logName, 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0);
 }
 
 static void taosCleanupArgs() {

source/libs/function/src/udfd.c

@@ -1564,8 +1564,8 @@ static void udfdPrintVersion() {
 }
 
 static int32_t udfdInitLog() {
-  char logName[12] = {0};
+  const char *logName = "udfdlog";
-  snprintf(logName, sizeof(logName), "%slog", "udfd");
+  TAOS_CHECK_RETURN(taosInitLogOutput(&logName));
   return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, NULL, 0);
 }
 

source/os/src/osDir.c

@@ -339,7 +339,7 @@ int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) {
   OS_PARAM_CHECK(dirname);
   OS_PARAM_CHECK(outname);
   wordexp_t full_path;
-  int32_t code = wordexp(dirname, &full_path, 0);
+  int32_t   code = wordexp(dirname, &full_path, 0);
   switch (code) {
     case 0:
       break;
@@ -347,7 +347,7 @@ int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) {
       wordfree(&full_path);
       // FALL THROUGH
     default:
-      return code;
+      return terrno = TSDB_CODE_INVALID_PARA;
   }
 
   if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) {

source/os/src/osEnv.c

@@ -89,7 +89,7 @@ int32_t osDefaultInit() {
   }
   tstrncpy(tsDataDir, TD_DATA_DIR_PATH, sizeof(tsDataDir));
   tstrncpy(tsLogDir, TD_LOG_DIR_PATH, sizeof(tsLogDir));
-  if (strlen(tsTempDir) == 0){
+  if (strlen(tsTempDir) == 0) {
     tstrncpy(tsTempDir, TD_TMP_DIR_PATH, sizeof(tsTempDir));
   }
 

source/util/src/tlog.c

@@ -51,6 +51,13 @@
 #define LOG_EDITION_FLG ("C")
 #endif
 
+typedef enum {
+  LOG_OUTPUT_FILE = 0,    // default
+  LOG_OUTPUT_STDOUT = 1,  // stdout set by -o option on the command line
+  LOG_OUTPUT_STDERR = 2,  // stderr set by -e option on the command line
+  LOG_OUTPUT_NULL = 4,    // /dev/null set by -o option on the command line
+} ELogOutputType;
+
 typedef struct {
   char         *buffer;
   int32_t       buffStart;
@@ -73,6 +80,7 @@ typedef struct {
   int32_t       openInProgress;
   int64_t       lastKeepFileSec;
   int64_t       timestampToday;
+  int8_t        outputType;  // ELogOutputType
   pid_t         pid;
   char          logName[PATH_MAX];
   char          slowLogName[PATH_MAX];
@@ -96,6 +104,8 @@ bool tsAssert = true;
 #endif
 int32_t tsNumOfLogLines = 10000000;
 int32_t tsLogKeepDays = 0;
+
+char   *tsLogOutput = NULL;
 LogFp   tsLogFp = NULL;
 int64_t tsNumOfErrorLogs = 0;
 int64_t tsNumOfInfoLogs = 0;
@@ -234,6 +244,58 @@ int32_t taosInitSlowLog() {
   return 0;
 }
 
+int32_t taosInitLogOutput(const char **ppLogName) {
+  const char *pLog = tsLogOutput;
+  const char *pLogName = NULL;
+  if (pLog) {
+    if (!tIsValidFilePath(pLog, NULL)) {
+      fprintf(stderr, "invalid log output destination:%s, contains illegal char\n", pLog);
+      return TSDB_CODE_INVALID_CFG;
+    }
+    if (0 == strcasecmp(pLog, "stdout")) {
+      tsLogObj.outputType = LOG_OUTPUT_STDOUT;
+      if (ppLogName) *ppLogName = pLog;
+      return 0;
+    }
+    if (0 == strcasecmp(pLog, "stderr")) {
+      tsLogObj.outputType = LOG_OUTPUT_STDERR;
+      if (ppLogName) *ppLogName = pLog;
+      return 0;
+    }
+    if (0 == strcasecmp(pLog, "/dev/null")) {
+      tsLogObj.outputType = LOG_OUTPUT_NULL;
+      if (ppLogName) *ppLogName = pLog;
+      return 0;
+    }
+    int32_t len = strlen(pLog);
+    if (len < 1) {
+      fprintf(stderr, "invalid log output destination:%s, should not be empty\n", pLog);
+      return TSDB_CODE_INVALID_CFG;
+    }
+    const char *p = pLog + (len - 1);
+    if (*p == '/' || *p == '\\') {
+      return 0;
+    }
+
+    if ((p = strrchr(pLog, '/')) || (p = strrchr(pLog, '\\'))) {
+      pLogName = p + 1;
+    } else {
+      pLogName = pLog;
+    }
+    if (strcmp(pLogName, ".") == 0 || strcmp(pLogName, "..") == 0) {
+      fprintf(stderr, "invalid log output destination:%s\n", pLog);
+      return TSDB_CODE_INVALID_CFG;
+    }
+
+    if (!tIsValidFileName(pLogName, NULL)) {
+      fprintf(stderr, "invalid log output destination:%s, contains illegal char\n", pLog);
+      return TSDB_CODE_INVALID_CFG;
+    }
+    if (ppLogName) *ppLogName = pLogName;
+  }
+  return 0;
+}
+
 int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) {
   if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0;
   int32_t code = osUpdate();
@@ -241,6 +303,11 @@ int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) {
     uError("failed to update os info, reason:%s", tstrerror(code));
   }
 
+  if (tsLogObj.outputType == LOG_OUTPUT_STDOUT || tsLogObj.outputType == LOG_OUTPUT_STDERR ||
+      tsLogObj.outputType == LOG_OUTPUT_NULL) {
+    return 0;
+  }
+
   TAOS_CHECK_RETURN(taosInitNormalLog(logName, maxFiles));
   if (tsc) {
     TAOS_CHECK_RETURN(taosInitSlowLog());
@@ -283,6 +350,7 @@ void taosCloseLog() {
     taosMemoryFreeClear(tsLogObj.logHandle);
     tsLogObj.logHandle = NULL;
   }
+  taosMemoryFreeClear(tsLogOutput);
 }
 
 static bool taosLockLogFile(TdFilePtr pFile) {
@@ -374,7 +442,7 @@ static OldFileKeeper *taosOpenNewFile() {
   }
 
   TdFilePtr pOldFile = tsLogObj.logHandle->pFile;
-  atomic_store_ptr(&tsLogObj.logHandle->pFile, pFile);
+  tsLogObj.logHandle->pFile = pFile;
   tsLogObj.lines = 0;
   tsLogObj.openInProgress = 0;
   OldFileKeeper *oldFileKeeper = taosMemoryMalloc(sizeof(OldFileKeeper));
@@ -666,7 +734,7 @@ static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *b
     if (tsAsyncLog) {
       TAOS_UNUSED(taosPushLogBuffer(tsLogObj.logHandle, buffer, len));
     } else {
-      TAOS_UNUSED(taosWriteFile(atomic_load_ptr(tsLogObj.logHandle->pFile), buffer, len));
+      TAOS_UNUSED(taosWriteFile(tsLogObj.logHandle->pFile, buffer, len));
     }
 
     if (tsNumOfLogLines > 0) {
@@ -680,10 +748,19 @@ static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *b
     }
   }
 
-  if (dflag & DEBUG_SCREEN) {
+  int fd = 0;
+  if (tsLogObj.outputType == LOG_OUTPUT_FILE) {
+    if (dflag & DEBUG_SCREEN) fd = 1;
+  } else if (tsLogObj.outputType == LOG_OUTPUT_STDOUT) {
+    fd = 1;
+  } else if (tsLogObj.outputType == LOG_OUTPUT_STDERR) {
+    fd = 2;
+  }
+
+  if (fd) {
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wunused-result"
-    if (write(1, buffer, (uint32_t)len) < 0) {
+    if (write(fd, buffer, (uint32_t)len) < 0) {
       TAOS_UNUSED(printf("failed to write log to screen, reason:%s\n", strerror(errno)));
     }
 #pragma GCC diagnostic pop

source/util/src/tutil.c

@@ -16,6 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "tutil.h"
 #include "tlog.h"
+#include "regex.h"
 
 void *tmemmem(const char *haystack, int32_t hlen, const char *needle, int32_t nlen) {
   const char *limit;
@@ -520,3 +521,29 @@ int32_t parseCfgReal(const char *str, float *out) {
   *out = val;
   return TSDB_CODE_SUCCESS;
 }
+
+bool tIsValidFileName(const char *fileName, const char *pattern) {
+  const char *fileNamePattern = "^[a-zA-Z0-9_.-]+$";
+
+  regex_t fileNameReg;
+
+  if (pattern) fileNamePattern = pattern;
+
+  if (regcomp(&fileNameReg, fileNamePattern, REG_EXTENDED) != 0) {
+    fprintf(stderr, "failed to compile file name pattern:%s\n", fileNamePattern);
+    return false;
+  }
+
+  int32_t code = regexec(&fileNameReg, fileName, 0, NULL, 0);
+  regfree(&fileNameReg);
+  if (code != 0) {
+    return false;
+  }
+  return true;
+}
+
+bool tIsValidFilePath(const char *filePath, const char *pattern) {
+  const char *filePathPattern = "^[a-zA-Z0-9/\\_.-]+$";
+
+  return tIsValidFileName(filePath, pattern ? pattern : filePathPattern);
+}

tools/shell/src/shellArguments.c

@@ -49,7 +49,12 @@
 #define SHELL_PKT_LEN  "Packet length used for net test, default is 1024 bytes."
 #define SHELL_PKT_NUM  "Packet numbers used for net test, default is 100."
 #define SHELL_BI_MODE  "Set BI mode"
-#define SHELL_VERSION  "Print program version."
+#define SHELL_LOG_OUTPUT                                                                                              \
+  "Specify log output. Options:\n\r\t\t\t     stdout, stderr, /dev/null, <directory>, <directory>/<filename>, "       \
+  "<filename>\n\r\t\t\t     * If OUTPUT contains an absolute directory, logs will be stored in that directory "       \
+  "instead of logDir.\n\r\t\t\t     * If OUTPUT contains a relative directory, logs will be stored in the directory " \
+  "combined with logDir and the relative directory."
+#define SHELL_VERSION "Print program version."
 
 #ifdef WEBSOCKET
 #define SHELL_DSN     "Use dsn to connect to the cloud server or to a remote server which provides WebSocket connection."
@@ -74,6 +79,7 @@ void shellPrintHelp() {
   printf("%s%s%s%s\r\n", indent, "-l,", indent, SHELL_PKT_LEN);
   printf("%s%s%s%s\r\n", indent, "-n,", indent, SHELL_NET_ROLE);
   printf("%s%s%s%s\r\n", indent, "-N,", indent, SHELL_PKT_NUM);
+  printf("%s%s%s%s\r\n", indent, "-o,", indent, SHELL_LOG_OUTPUT);
   printf("%s%s%s%s\r\n", indent, "-p,", indent, SHELL_PASSWORD);
   printf("%s%s%s%s\r\n", indent, "-P,", indent, SHELL_PORT);
   printf("%s%s%s%s\r\n", indent, "-r,", indent, SHELL_RAW_TIME);
@@ -134,6 +140,7 @@ static struct argp_option shellOptions[] = {
 #endif
     {"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM},
     {"bimode", 'B', 0, 0, SHELL_BI_MODE},
+    {"log-output", 'o', "OUTPUT", 0, SHELL_LOG_OUTPUT},
     {0},
 };
 
@@ -152,6 +159,8 @@ static void shellParseArgsUseArgp(int argc, char *argv[]) {
 #define ARGP_ERR_UNKNOWN E2BIG
 #endif
 
+extern char* tsLogOutput;
+
 static int32_t shellParseSingleOpt(int32_t key, char *arg) {
   SShellArgs *pArgs = &shell.args;
 
@@ -222,6 +231,21 @@ static int32_t shellParseSingleOpt(int32_t key, char *arg) {
     case 'N':
       pArgs->pktNum = atoi(arg);
       break;
+    case 'o':
+      if (strlen(arg) >= PATH_MAX) {
+        printf("failed to set log output since length overflow, max length is %d\n", PATH_MAX);
+        return TSDB_CODE_INVALID_CFG;
+      }
+      tsLogOutput = taosMemoryMalloc(PATH_MAX);
+      if (!tsLogOutput) {
+        printf("failed to set log output: '%s' since %s\n", arg, tstrerror(terrno));
+        return terrno;
+      }
+      if (taosExpandDir(arg, tsLogOutput, PATH_MAX) != 0) {
+        printf("failed to expand log output: '%s' since %s\n", arg, tstrerror(terrno));
+        return terrno;
+      }
+      break;
 #ifdef WEBSOCKET
     case 'R':
       pArgs->restful = true;