test: add test case for log output

source/common/src/tglobal.c

@@ -1032,6 +1032,9 @@ static int32_t taosSetLogOutput(SConfig *pCfg) {
         }
         TAOS_CHECK_RETURN(cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true));
       }
+    } else {
+      tstrncpy(tsLogDir, pLog, PATH_MAX);
+      TAOS_CHECK_RETURN(cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true));
     }
   }
   return 0;

source/dnode/mgmt/exe/dmMain.c

@@ -240,23 +240,28 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
       }
     } else if (strcmp(argv[i], "-k") == 0) {
       global.generateGrant = true;
-    } else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--log-output") == 0) {
+    } else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--log-output") == 0 ||
-      if (i < argc - 1) {
+               strncmp(argv[i], "--log-output=", 13) == 0) {
-        if (strlen(argv[++i]) >= PATH_MAX) {
+      if ((i < argc - 1) || ((i == argc - 1) && strncmp(argv[i], "--log-output=", 13) == 0)) {
-          printf("failed to set log output since length overflow, max length is %d\n", PATH_MAX);
+        int32_t     klen = strlen(argv[i]);
+        int32_t     vlen = klen < 13 ? strlen(argv[++i]) : klen - 13;
+        const char *val = argv[i];
+        if (klen >= 13) val += 13;
+        if (vlen <= 0 || vlen >= PATH_MAX) {
+          printf("failed to set log output since invalid vlen:%d, valid range: [1, %d)\n", vlen, PATH_MAX);
           return TSDB_CODE_INVALID_CFG;
         }
         tsLogOutput = taosMemoryMalloc(PATH_MAX);
         if (!tsLogOutput) {
-          printf("failed to set log output: '%s' since %s\n", argv[i], tstrerror(terrno));
+          printf("failed to set log output: '%s' since %s\n", val, tstrerror(terrno));
           return terrno;
         }
-        if (taosExpandDir(argv[i], tsLogOutput, PATH_MAX) != 0) {
+        if (taosExpandDir(val, tsLogOutput, PATH_MAX) != 0) {
-          printf("failed to expand log output: '%s' since %s\n", argv[i], tstrerror(terrno));
+          printf("failed to expand log output: '%s' since %s\n", val, tstrerror(terrno));
           return terrno;
         }
       } else {
-        printf("'-o' requires a parameter\n");
+        printf("'%s' requires a parameter\n", argv[i]);
         return TSDB_CODE_INVALID_CFG;
       }
     } else if (strcmp(argv[i], "-y") == 0) {
@@ -402,6 +407,7 @@ int mainWindows(int argc, char **argv) {
   int32_t code = 0;
 #endif
 
+  char *p = taosMemoryCalloc(10, 10);
   if (global.generateGrant) {
     dmGenerateGrant();
     taosCleanupArgs();

source/util/src/tlog.c

@@ -1078,10 +1078,12 @@ static void taosWriteLog(SLogBuff *pLogBuf) {
 }
 
 #define LOG_ROTATE_INTERVAL 1800
-#ifdef BUILD_TEST
+#ifdef ASSERT_NOT_CORE
-#define LOG_ROTATE_BOOT 5
+#define LOG_INACTIVE_TIME 3600
+#define LOG_ROTATE_BOOT   5
 #else
-#define LOG_ROTATE_BOOT 180
+#define LOG_INACTIVE_TIME 5
+#define LOG_ROTATE_BOOT   (LOG_INACTIVE_TIME + 1)
 #endif
 
 static void *taosLogRotateFunc(void *param) {
@@ -1118,7 +1120,7 @@ static void *taosLogRotateFunc(void *param) {
     }
 
     int64_t fileSec = 0;
-    if ((code = taosStr2int64(pSec + 1, NULL)) != 0) {
+    if ((code = taosStr2int64(pSec + 1, &fileSec)) != 0) {
       uWarn("%s:%d failed to convert %s to int64 since %s", __func__, __LINE__, pSec + 1, tstrerror(code));
       continue;
     }
@@ -1132,19 +1134,19 @@ static void *taosLogRotateFunc(void *param) {
       continue;
     }
 
-    int64_t elapseSec = taosGetTimestampMs() / 1000 - mtime;
+    int64_t inactiveSec = taosGetTimestampMs() / 1000 - mtime;
 
-    if (elapseSec < 7200) {
+    if (inactiveSec < LOG_INACTIVE_TIME) {
       continue;
     }
 
     char fullName[PATH_MAX] = {0};
     snprintf(fullName, sizeof(fullName), "%s%s%s", tsLogDir, TD_DIRSEP, fname);
 
-    int32_t days = elapseSec / 86400 + 1;
+    int32_t days = inactiveSec / 86400 + 1;
     if (tsLogKeepDays > 0 && days > tsLogKeepDays) {
       TAOS_UNUSED(taosRemoveFile(fullName));
-      uInfo("file:%s is removed, days:%d keepDays:%d, sed:%" PRId64, fullName, days, tsLogKeepDays, fileSec);
+      uInfo("file:%s is removed, days:%d, keepDays:%d, sed:%" PRId64, fullName, days, tsLogKeepDays, fileSec);
     } else {
       taosKeepOldLog(fullName);  // compress
     }

tests/parallel_test/cases.task

@@ -352,6 +352,7 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/telemetry.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/backquote_check.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdMonitor.py
+,,n,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdlog.py
 ,,n,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3
 ,,n,system-test,python3 ./test.py -f 0-others/udfTest.py
 ,,n,system-test,python3 ./test.py -f 0-others/udf_create.py

tests/system-test/0-others/taosdlog.py

@@ -2,11 +2,18 @@ import taos
 import sys
 import time
 import os
+import os.path
+import platform
+import concurrent.futures
+import shutil  # Add this import
 
 from util.log import *
 from util.sql import *
 from util.cases import *
 from util.dnodes import *
+import time
+import socket
+import subprocess
 
 class TDTestCase:
 
@@ -20,8 +27,13 @@ class TDTestCase:
             tdLog.exit("taosd not found!")
         else:
             tdLog.info("taosd found in %s" % self.buildPath)
+        self.taosdPath = self.buildPath + "/build/bin/taosd"
+        self.taosPath = self.buildPath + "/build/bin/taos"
         self.logPath = self.buildPath + "/../sim/dnode1/log"
+        tdLog.info("taosd path: %s" % self.taosdPath)
+        tdLog.info("taos path: %s" % self.taosPath)
         tdLog.info("log path: %s" % self.logPath)
+        self.commonCfgDict = {}
 
     def getBuildPath(self):
         selfPath = os.path.dirname(os.path.realpath(__file__))
@@ -39,60 +51,272 @@ class TDTestCase:
                     break
         return buildPath
 
-    def logPathBasic(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
-        tdSql.query("create user testpy pass 'testpy'")
-
-        tdDnodes.stop(1)
-        time.sleep(2)
-        tdSql.error("select * from information_schema.ins_databases")
-        tdSql.checkRows(2)
-        os.system("rm -rf  %s" % self.logPath)
-        if os.path.exists(self.logPath) == True:
-            tdLog.exit("log path still exist!")
-
-        tdDnodes.start(1)
-        time.sleep(2)
-        if os.path.exists(self.logPath) != True:
-            tdLog.exit("log path is not generated!")
-
-        tdSql.query("select * from information_schema.ins_databases")
-        tdSql.checkRows(2)
-
     def prepareCfg(self, cfgPath, cfgDict):
+        tdLog.info("make dir %s" % cfgPath)
+        os.makedirs(cfgPath, exist_ok=True)
         with open(cfgPath + "/taos.cfg", "w") as f:
+            for key in self.commonCfgDict:
+                f.write("%s %s\n" % (key, self.commonCfgDict[key]))
             for key in cfgDict:
                 f.write("%s %s\n" % (key, cfgDict[key]))
+        if not os.path.exists(cfgPath + "/taos.cfg"):
+            tdLog.exit("taos.cfg not found in %s" % cfgPath)
+        else:
+            tdLog.info("taos.cfg found in %s" % cfgPath)
 
-    def check_function_1(self):
+    def checkProcessPid(self,processName):
-        # Implementation of check function 1
+        i=0
-        tdLog.info("Running check function 1")
+        while i<60:
-        # Add your check logic here
+            tdLog.info(f"wait stop {processName}")
+            processPid = subprocess.getstatusoutput(f'ps aux|grep {processName} |grep -v "grep"|awk \'{{print $2}}\'')[1]
+            tdLog.info(f"times:{i},{processName}-pid:{processPid}")
+            if(processPid == ""):
+                break
+            i += 1
+            time.sleep(1)
+        else:
+            tdLog.info(f'this processName is not stoped in 60s')
 
-    def check_function_2(self):
+    def closeTaosd(self, signal=9):
-        # Implementation of check function 2
+        tdLog.info("Closing taosd")
-        tdLog.info("Running check function 2")
+        if platform.system().lower() == 'windows':
-        # Add your check logic here
+            psCmd = "for /f %%a in ('wmic process where \"name='taosd.exe' and CommandLine like '%%dnode%d%%'\" get processId ^| xargs echo ^| awk ^'{print $2}^' ^&^& echo aa') do @(ps | grep %%a | awk '{print $1}' | xargs)" % (self.index)
+        else:
+            psCmd = "ps -ef | grep -w taosd | grep -v grep | awk '{print $2}' | xargs"
+        tdLog.info(f"psCmd:{psCmd}")
 
-    def check_function_3(self):
+        try:
-        # Implementation of check function 3
+            rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        tdLog.info("Running check function 3")
+            processID = rem.stdout.decode().strip()
-        # Add your check logic here
+            tdLog.info(f"processID:{processID}")
+        except Exception as e:
+            tdLog.info(f"closeTaosd error:{e}")
+            processID = ""
+        tdLog.info(f"processID:{processID}")
+        onlyKillOnceWindows = 0
+        while(processID):
+            if not platform.system().lower() == 'windows' or (onlyKillOnceWindows == 0 and platform.system().lower() == 'windows'):
+                killCmd = "kill -%d %s > /dev/null 2>&1" % (signal, processID)
+                if platform.system().lower() == 'windows':
+                    killCmd = "kill -%d %s > nul 2>&1" % (signal, processID)
+                tdLog.info(f"killCmd:{killCmd}")
+                os.system(killCmd)
+                tdLog.info(f"killed taosd process {processID}")
+                onlyKillOnceWindows = 1
+            time.sleep(1)
+            try:
+                rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                processID = rem.stdout.decode().strip()
+            except Exception as e:
+                tdLog.info(f"closeTaosd error:{e}")
+                processID = ""
 
-    def prepareCheckFunctions(self):
+    def closeTaos(self, signal=9):
+        tdLog.info("Closing taos")
+        if platform.system().lower() == 'windows':
+            psCmd = "for /f %%a in ('wmic process where \"name='taos.exe'\" get processId ^| xargs echo ^| awk ^'{print $2}^' ^&^& echo aa') do @(ps | grep %%a | awk '{print $1}' | xargs)"
+        else:
+            psCmd = "ps -ef | grep -w taos | grep -v grep | awk '{print $2}'"
+        tdLog.info(f"psCmd:{psCmd}")
+
+        try:
+            rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            processID = rem.stdout.decode().strip()
+            tdLog.info(f"processID:{processID}")
+        except Exception as e:
+            tdLog.info(f"closeTaos error:{e}")
+            processID = ""
+        tdLog.info(f"processID:{processID}")
+        onlyKillOnceWindows = 0
+        while(processID):
+            if not platform.system().lower() == 'windows' or (onlyKillOnceWindows == 0 and platform.system().lower() == 'windows'):
+                killCmd = "kill -%d %s > /dev/null 2>&1" % (signal, processID)
+                if platform.system().lower() == 'windows':
+                    killCmd = "kill -%d %s > nul 2>&1" % (signal, processID)
+                tdLog.info(f"killCmd:{killCmd}")
+                os.system(killCmd)
+                tdLog.info(f"killed taos process {processID}")
+                onlyKillOnceWindows = 1
+            time.sleep(1)
+            try:
+                rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                processID = rem.stdout.decode().strip()
+            except Exception as e:
+                tdLog.info(f"closeTaos error:{e}")
+                processID = ""
+
+    def openBin(self, binPath, waitSec=5):
+        tdLog.info(f"Opening {binPath}")
+        try:
+            process = subprocess.Popen(binPath, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            time.sleep(waitSec)
+            if process.poll() is None:
+                tdLog.info(f"{binPath} started successfully")
+            else:
+                error = process.stderr.read().decode(encoding="utf-8").strip()
+                raise Exception(f"Failed to start {binPath}: {error}")
+        except Exception as e:
+            raise Exception(f"Failed to start {binPath}: %s" % repr(e))
+
+    def openTaosd(self, args="", waitSec=8):
+        self.openBin(f'{self.taosdPath} {args}', waitSec)
+
+    def openTaos(self, args="", waitSec=3):
+        self.openBin(f'{self.taosPath} {args}', waitSec)
+
+
+    def prepare_logoutput(self, desc, port, logOutput):
+        tdLog.info("Preparing %s, port:%s" % (desc, port))
+        dnodePath = self.buildPath + "/../sim/dnode%s" % port
+        tdLog.info("remove dnodePath:%s" % dnodePath)
+        try:
+            shutil.rmtree(dnodePath)
+        except Exception as e:
+            tdLog.info(f"Failed to remove directory {dnodePath}: {e}")
+        try:
+            self.prepareCfg(dnodePath, {"serverPort": port,
+                "dataDir": dnodePath + os.sep + "data",
+                "logDir": dnodePath + os.sep + "log"})
+        except Exception as e:
+            tdLog.info(f"Failed to prepare configuration for {dnodePath}: {e}")
+            return
+        try:
+            self.openTaosd(f"-c {dnodePath} {logOutput}")
+            self.openTaos(f"-c {dnodePath} {logOutput}")
+        except Exception as e:
+            tdLog.info(f"Failed to open taosd or taos for {dnodePath}: {e}")
+
+    def prepare_stdout(self):
+        list = self.prepare_list[0]
+        self.prepare_logoutput(list[0], list[1], "-o " + list[0])
+
+    def check_stdout(self):
+        tdLog.info("Running check stdout")
+        dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[0][1]
+        tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0"))
+        tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0"))
+
+    def prepare_stderr(self):
+        list = self.prepare_list[1]
+        self.prepare_logoutput(list[0], list[1], "--log-output " + list[0])
+
+    def check_stderr(self):
+        tdLog.info("Running check stderr")
+        dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[1][1]
+        tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0"))
+        tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0"))
+
+    def prepare_dev_null(self):
+        list = self.prepare_list[2]
+        self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0])
+
+    def check_dev_null(self):
+        tdLog.info("Running check /dev/null")
+        dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[2][1]
+        tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0"))
+        tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0"))
+    
+    def prepare_fullpath(self):
+        list = self.prepare_list[3]
+        dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[3][1]
+        self.prepare_logoutput(list[0], list[1], "-o " + dnodePath + "/log0/" )
+    
+    def check_fullpath(self):
+        tdLog.info("Running check fullpath")
+        logPath = self.buildPath + "/../sim/dnode%s/log0/" % self.prepare_list[3][1]
+        tdSql.checkEqual(True, os.path.exists(f"{logPath}taosdlog.0"))
+        tdSql.checkEqual(True, os.path.exists(f"{logPath}taoslog0.0"))
+    
+    def prepare_fullname(self):
+        list = self.prepare_list[4]
+        dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[4][1]
+        self.prepare_logoutput(list[0], list[1], "--log-output " + dnodePath + "/log0/" + list[0])
+        dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[4][1]
+
+    def check_fullname(self):
+        tdLog.info("Running check fullname")
+        logPath = self.buildPath + "/../sim/dnode%s/log0/" % self.prepare_list[4][1]
+        tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[4][0] + ".0"))
+        tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[4][0] + "0.0"))
+    
+    def prepare_relativepath(self):
+        list = self.prepare_list[5]
+        self.prepare_logoutput(list[0], list[1], "--log-output=" + "log0/")
+    
+    def check_relativepath(self):
+        tdLog.info("Running check relativepath")
+        logPath = self.buildPath + "/../sim/dnode%s/log/log0/" % self.prepare_list[5][1]
+        tdSql.checkEqual(True, os.path.exists(logPath + "taosdlog.0"))
+        tdSql.checkEqual(True, os.path.exists(logPath + "taoslog0.0"))
+    
+    def prepare_relativename(self):
+        list = self.prepare_list[6]
+        self.prepare_logoutput(list[0], list[1], "-o " + "log0/" + list[0])
+    def check_relativename(self):
+        tdLog.info("Running check relativename")
+        logPath = self.buildPath + "/../sim/dnode%s/log/log0/" % self.prepare_list[6][1]
+        tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[6][0] + ".0"))
+        tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[6][0] + "0.0"))
+    
+    def prepare_filename(self):
+        list = self.prepare_list[7]
+        self.prepare_logoutput(list[0], list[1], "--log-output " + list[0])
+    def check_filename(self):
+        tdLog.info("Running check filename")
+        logPath = self.buildPath + "/../sim/dnode%s/log/" % self.prepare_list[7][1]
+        tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[7][0] + ".0"))
+        tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[7][0] + "0.0"))
+    
+    def prepare_empty(self):
+        list = self.prepare_list[8]
+        self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0])
+    def check_empty(self):
+        tdLog.info("Running check empty")
+        logPath = self.buildPath + "/../sim/dnode%s/log" % self.prepare_list[8][1]
+        tdSql.checkEqual(False, os.path.exists(f"{logPath}/taosdlog.0"))
+        tdSql.checkEqual(False, os.path.exists(f"{logPath}/taoslog.0"))
+
+    def prepareCheckResources(self):
+        self.prepare_list = [["stdout", "10030"], ["stderr", "10031"], ["/dev/null", "10032"], 
+                             ["fullpath", "10033"], ["fullname", "10034"], ["relativepath", "10035"], 
+                             ["relativename", "10036"], ["filename", "10037"], ["empty", "10038"]]
         self.check_functions = {
-            "check_function_1": self.check_function_1,
+            self.prepare_stdout: self.check_stdout,
-            "check_function_2": self.check_function_2,
+            self.prepare_stderr: self.check_stderr,
-            "check_function_3": self.check_function_3
+            self.prepare_dev_null: self.check_dev_null,
+            self.prepare_fullpath: self.check_fullpath,
+            self.prepare_fullname: self.check_fullname,
+            self.prepare_relativepath: self.check_relativepath,
+            self.prepare_relativename: self.check_relativename,
+            self.prepare_filename: self.check_filename,
+            self.prepare_empty: self.check_empty,
         }
 
     def checkLogOutput(self):
-        self.prepareCheckFunctions()
+        self.closeTaosd()
-        for key, check_func in self.check_functions.items():
+        self.closeTaos()
-            print(f"Running {key}")
+        self.prepareCheckResources()
-            check_func()
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            prepare_futures = [executor.submit(prepare_func) for prepare_func, _ in self.check_functions.items()]
+            for future in concurrent.futures.as_completed(prepare_futures):
+                try:
+                    future.result()
+                except Exception as e:
+                    raise Exception(f"Error in prepare function: {e}")
+            
+            check_futures = [executor.submit(check_func) for _, check_func in self.check_functions.items()]
+            for future in concurrent.futures.as_completed(check_futures):
+                try:
+                    future.result()
+                except Exception as e:
+                    raise Exception(f"Error in prepare function: {e}")
+        self.closeTaosd()
+        self.closeTaos()
+
+    def is_windows(self):
+        return os.name == 'nt'
 
     def run(self):
-        # self.logPathBasic()
         self.checkLogOutput()
 
     def stop(self):