diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c1a3026394..62a1e3f500 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1032,6 +1032,9 @@ static int32_t taosSetLogOutput(SConfig *pCfg) { } TAOS_CHECK_RETURN(cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true)); } + } else { + tstrncpy(tsLogDir, pLog, PATH_MAX); + TAOS_CHECK_RETURN(cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true)); } } return 0; diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index e15e946d5a..d270699e58 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -240,23 +240,28 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { } } else if (strcmp(argv[i], "-k") == 0) { global.generateGrant = true; - } else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--log-output") == 0) { - if (i < argc - 1) { - if (strlen(argv[++i]) >= PATH_MAX) { - printf("failed to set log output since length overflow, max length is %d\n", PATH_MAX); + } else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--log-output") == 0 || + strncmp(argv[i], "--log-output=", 13) == 0) { + if ((i < argc - 1) || ((i == argc - 1) && strncmp(argv[i], "--log-output=", 13) == 0)) { + int32_t klen = strlen(argv[i]); + int32_t vlen = klen < 13 ? strlen(argv[++i]) : klen - 13; + const char *val = argv[i]; + if (klen >= 13) val += 13; + if (vlen <= 0 || vlen >= PATH_MAX) { + printf("failed to set log output since invalid vlen:%d, valid range: [1, %d)\n", vlen, PATH_MAX); return TSDB_CODE_INVALID_CFG; } tsLogOutput = taosMemoryMalloc(PATH_MAX); if (!tsLogOutput) { - printf("failed to set log output: '%s' since %s\n", argv[i], tstrerror(terrno)); + printf("failed to set log output: '%s' since %s\n", val, tstrerror(terrno)); return terrno; } - if (taosExpandDir(argv[i], tsLogOutput, PATH_MAX) != 0) { - printf("failed to expand log output: '%s' since %s\n", argv[i], tstrerror(terrno)); + if (taosExpandDir(val, tsLogOutput, PATH_MAX) != 0) { + printf("failed to expand log output: '%s' since %s\n", val, tstrerror(terrno)); return terrno; } } else { - printf("'-o' requires a parameter\n"); + printf("'%s' requires a parameter\n", argv[i]); return TSDB_CODE_INVALID_CFG; } } else if (strcmp(argv[i], "-y") == 0) { @@ -402,6 +407,7 @@ int mainWindows(int argc, char **argv) { int32_t code = 0; #endif + char *p = taosMemoryCalloc(10, 10); if (global.generateGrant) { dmGenerateGrant(); taosCleanupArgs(); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 9678802a01..707923f200 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -1078,10 +1078,12 @@ static void taosWriteLog(SLogBuff *pLogBuf) { } #define LOG_ROTATE_INTERVAL 1800 -#ifdef BUILD_TEST -#define LOG_ROTATE_BOOT 5 +#ifdef ASSERT_NOT_CORE +#define LOG_INACTIVE_TIME 3600 +#define LOG_ROTATE_BOOT 5 #else -#define LOG_ROTATE_BOOT 180 +#define LOG_INACTIVE_TIME 5 +#define LOG_ROTATE_BOOT (LOG_INACTIVE_TIME + 1) #endif static void *taosLogRotateFunc(void *param) { @@ -1118,7 +1120,7 @@ static void *taosLogRotateFunc(void *param) { } int64_t fileSec = 0; - if ((code = taosStr2int64(pSec + 1, NULL)) != 0) { + if ((code = taosStr2int64(pSec + 1, &fileSec)) != 0) { uWarn("%s:%d failed to convert %s to int64 since %s", __func__, __LINE__, pSec + 1, tstrerror(code)); continue; } @@ -1132,19 +1134,19 @@ static void *taosLogRotateFunc(void *param) { continue; } - int64_t elapseSec = taosGetTimestampMs() / 1000 - mtime; + int64_t inactiveSec = taosGetTimestampMs() / 1000 - mtime; - if (elapseSec < 7200) { + if (inactiveSec < LOG_INACTIVE_TIME) { continue; } char fullName[PATH_MAX] = {0}; snprintf(fullName, sizeof(fullName), "%s%s%s", tsLogDir, TD_DIRSEP, fname); - int32_t days = elapseSec / 86400 + 1; + int32_t days = inactiveSec / 86400 + 1; if (tsLogKeepDays > 0 && days > tsLogKeepDays) { TAOS_UNUSED(taosRemoveFile(fullName)); - uInfo("file:%s is removed, days:%d keepDays:%d, sed:%" PRId64, fullName, days, tsLogKeepDays, fileSec); + uInfo("file:%s is removed, days:%d, keepDays:%d, sed:%" PRId64, fullName, days, tsLogKeepDays, fileSec); } else { taosKeepOldLog(fullName); // compress } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index db0339381e..ac33c38545 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -352,6 +352,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/telemetry.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/backquote_check.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdMonitor.py +,,n,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdlog.py ,,n,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 ,,n,system-test,python3 ./test.py -f 0-others/udfTest.py ,,n,system-test,python3 ./test.py -f 0-others/udf_create.py diff --git a/tests/system-test/0-others/taosdlog.py b/tests/system-test/0-others/taosdlog.py index 604617087f..a0981d7a72 100644 --- a/tests/system-test/0-others/taosdlog.py +++ b/tests/system-test/0-others/taosdlog.py @@ -2,11 +2,18 @@ import taos import sys import time import os +import os.path +import platform +import concurrent.futures +import shutil # Add this import from util.log import * from util.sql import * from util.cases import * from util.dnodes import * +import time +import socket +import subprocess class TDTestCase: @@ -20,8 +27,13 @@ class TDTestCase: tdLog.exit("taosd not found!") else: tdLog.info("taosd found in %s" % self.buildPath) + self.taosdPath = self.buildPath + "/build/bin/taosd" + self.taosPath = self.buildPath + "/build/bin/taos" self.logPath = self.buildPath + "/../sim/dnode1/log" + tdLog.info("taosd path: %s" % self.taosdPath) + tdLog.info("taos path: %s" % self.taosPath) tdLog.info("log path: %s" % self.logPath) + self.commonCfgDict = {} def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -39,60 +51,272 @@ class TDTestCase: break return buildPath - def logPathBasic(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring - tdSql.query("create user testpy pass 'testpy'") - - tdDnodes.stop(1) - time.sleep(2) - tdSql.error("select * from information_schema.ins_databases") - tdSql.checkRows(2) - os.system("rm -rf %s" % self.logPath) - if os.path.exists(self.logPath) == True: - tdLog.exit("log path still exist!") - - tdDnodes.start(1) - time.sleep(2) - if os.path.exists(self.logPath) != True: - tdLog.exit("log path is not generated!") - - tdSql.query("select * from information_schema.ins_databases") - tdSql.checkRows(2) - def prepareCfg(self, cfgPath, cfgDict): + tdLog.info("make dir %s" % cfgPath) + os.makedirs(cfgPath, exist_ok=True) with open(cfgPath + "/taos.cfg", "w") as f: + for key in self.commonCfgDict: + f.write("%s %s\n" % (key, self.commonCfgDict[key])) for key in cfgDict: f.write("%s %s\n" % (key, cfgDict[key])) + if not os.path.exists(cfgPath + "/taos.cfg"): + tdLog.exit("taos.cfg not found in %s" % cfgPath) + else: + tdLog.info("taos.cfg found in %s" % cfgPath) - def check_function_1(self): - # Implementation of check function 1 - tdLog.info("Running check function 1") - # Add your check logic here + def checkProcessPid(self,processName): + i=0 + while i<60: + tdLog.info(f"wait stop {processName}") + processPid = subprocess.getstatusoutput(f'ps aux|grep {processName} |grep -v "grep"|awk \'{{print $2}}\'')[1] + tdLog.info(f"times:{i},{processName}-pid:{processPid}") + if(processPid == ""): + break + i += 1 + time.sleep(1) + else: + tdLog.info(f'this processName is not stoped in 60s') - def check_function_2(self): - # Implementation of check function 2 - tdLog.info("Running check function 2") - # Add your check logic here + def closeTaosd(self, signal=9): + tdLog.info("Closing taosd") + if platform.system().lower() == 'windows': + psCmd = "for /f %%a in ('wmic process where \"name='taosd.exe' and CommandLine like '%%dnode%d%%'\" get processId ^| xargs echo ^| awk ^'{print $2}^' ^&^& echo aa') do @(ps | grep %%a | awk '{print $1}' | xargs)" % (self.index) + else: + psCmd = "ps -ef | grep -w taosd | grep -v grep | awk '{print $2}' | xargs" + tdLog.info(f"psCmd:{psCmd}") - def check_function_3(self): - # Implementation of check function 3 - tdLog.info("Running check function 3") - # Add your check logic here + try: + rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + processID = rem.stdout.decode().strip() + tdLog.info(f"processID:{processID}") + except Exception as e: + tdLog.info(f"closeTaosd error:{e}") + processID = "" + tdLog.info(f"processID:{processID}") + onlyKillOnceWindows = 0 + while(processID): + if not platform.system().lower() == 'windows' or (onlyKillOnceWindows == 0 and platform.system().lower() == 'windows'): + killCmd = "kill -%d %s > /dev/null 2>&1" % (signal, processID) + if platform.system().lower() == 'windows': + killCmd = "kill -%d %s > nul 2>&1" % (signal, processID) + tdLog.info(f"killCmd:{killCmd}") + os.system(killCmd) + tdLog.info(f"killed taosd process {processID}") + onlyKillOnceWindows = 1 + time.sleep(1) + try: + rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + processID = rem.stdout.decode().strip() + except Exception as e: + tdLog.info(f"closeTaosd error:{e}") + processID = "" - def prepareCheckFunctions(self): + def closeTaos(self, signal=9): + tdLog.info("Closing taos") + if platform.system().lower() == 'windows': + psCmd = "for /f %%a in ('wmic process where \"name='taos.exe'\" get processId ^| xargs echo ^| awk ^'{print $2}^' ^&^& echo aa') do @(ps | grep %%a | awk '{print $1}' | xargs)" + else: + psCmd = "ps -ef | grep -w taos | grep -v grep | awk '{print $2}'" + tdLog.info(f"psCmd:{psCmd}") + + try: + rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + processID = rem.stdout.decode().strip() + tdLog.info(f"processID:{processID}") + except Exception as e: + tdLog.info(f"closeTaos error:{e}") + processID = "" + tdLog.info(f"processID:{processID}") + onlyKillOnceWindows = 0 + while(processID): + if not platform.system().lower() == 'windows' or (onlyKillOnceWindows == 0 and platform.system().lower() == 'windows'): + killCmd = "kill -%d %s > /dev/null 2>&1" % (signal, processID) + if platform.system().lower() == 'windows': + killCmd = "kill -%d %s > nul 2>&1" % (signal, processID) + tdLog.info(f"killCmd:{killCmd}") + os.system(killCmd) + tdLog.info(f"killed taos process {processID}") + onlyKillOnceWindows = 1 + time.sleep(1) + try: + rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + processID = rem.stdout.decode().strip() + except Exception as e: + tdLog.info(f"closeTaos error:{e}") + processID = "" + + def openBin(self, binPath, waitSec=5): + tdLog.info(f"Opening {binPath}") + try: + process = subprocess.Popen(binPath, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + time.sleep(waitSec) + if process.poll() is None: + tdLog.info(f"{binPath} started successfully") + else: + error = process.stderr.read().decode(encoding="utf-8").strip() + raise Exception(f"Failed to start {binPath}: {error}") + except Exception as e: + raise Exception(f"Failed to start {binPath}: %s" % repr(e)) + + def openTaosd(self, args="", waitSec=8): + self.openBin(f'{self.taosdPath} {args}', waitSec) + + def openTaos(self, args="", waitSec=3): + self.openBin(f'{self.taosPath} {args}', waitSec) + + + def prepare_logoutput(self, desc, port, logOutput): + tdLog.info("Preparing %s, port:%s" % (desc, port)) + dnodePath = self.buildPath + "/../sim/dnode%s" % port + tdLog.info("remove dnodePath:%s" % dnodePath) + try: + shutil.rmtree(dnodePath) + except Exception as e: + tdLog.info(f"Failed to remove directory {dnodePath}: {e}") + try: + self.prepareCfg(dnodePath, {"serverPort": port, + "dataDir": dnodePath + os.sep + "data", + "logDir": dnodePath + os.sep + "log"}) + except Exception as e: + tdLog.info(f"Failed to prepare configuration for {dnodePath}: {e}") + return + try: + self.openTaosd(f"-c {dnodePath} {logOutput}") + self.openTaos(f"-c {dnodePath} {logOutput}") + except Exception as e: + tdLog.info(f"Failed to open taosd or taos for {dnodePath}: {e}") + + def prepare_stdout(self): + list = self.prepare_list[0] + self.prepare_logoutput(list[0], list[1], "-o " + list[0]) + + def check_stdout(self): + tdLog.info("Running check stdout") + dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[0][1] + tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0")) + tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0")) + + def prepare_stderr(self): + list = self.prepare_list[1] + self.prepare_logoutput(list[0], list[1], "--log-output " + list[0]) + + def check_stderr(self): + tdLog.info("Running check stderr") + dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[1][1] + tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0")) + tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0")) + + def prepare_dev_null(self): + list = self.prepare_list[2] + self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0]) + + def check_dev_null(self): + tdLog.info("Running check /dev/null") + dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[2][1] + tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0")) + tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0")) + + def prepare_fullpath(self): + list = self.prepare_list[3] + dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[3][1] + self.prepare_logoutput(list[0], list[1], "-o " + dnodePath + "/log0/" ) + + def check_fullpath(self): + tdLog.info("Running check fullpath") + logPath = self.buildPath + "/../sim/dnode%s/log0/" % self.prepare_list[3][1] + tdSql.checkEqual(True, os.path.exists(f"{logPath}taosdlog.0")) + tdSql.checkEqual(True, os.path.exists(f"{logPath}taoslog0.0")) + + def prepare_fullname(self): + list = self.prepare_list[4] + dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[4][1] + self.prepare_logoutput(list[0], list[1], "--log-output " + dnodePath + "/log0/" + list[0]) + dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[4][1] + + def check_fullname(self): + tdLog.info("Running check fullname") + logPath = self.buildPath + "/../sim/dnode%s/log0/" % self.prepare_list[4][1] + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[4][0] + ".0")) + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[4][0] + "0.0")) + + def prepare_relativepath(self): + list = self.prepare_list[5] + self.prepare_logoutput(list[0], list[1], "--log-output=" + "log0/") + + def check_relativepath(self): + tdLog.info("Running check relativepath") + logPath = self.buildPath + "/../sim/dnode%s/log/log0/" % self.prepare_list[5][1] + tdSql.checkEqual(True, os.path.exists(logPath + "taosdlog.0")) + tdSql.checkEqual(True, os.path.exists(logPath + "taoslog0.0")) + + def prepare_relativename(self): + list = self.prepare_list[6] + self.prepare_logoutput(list[0], list[1], "-o " + "log0/" + list[0]) + def check_relativename(self): + tdLog.info("Running check relativename") + logPath = self.buildPath + "/../sim/dnode%s/log/log0/" % self.prepare_list[6][1] + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[6][0] + ".0")) + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[6][0] + "0.0")) + + def prepare_filename(self): + list = self.prepare_list[7] + self.prepare_logoutput(list[0], list[1], "--log-output " + list[0]) + def check_filename(self): + tdLog.info("Running check filename") + logPath = self.buildPath + "/../sim/dnode%s/log/" % self.prepare_list[7][1] + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[7][0] + ".0")) + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[7][0] + "0.0")) + + def prepare_empty(self): + list = self.prepare_list[8] + self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0]) + def check_empty(self): + tdLog.info("Running check empty") + logPath = self.buildPath + "/../sim/dnode%s/log" % self.prepare_list[8][1] + tdSql.checkEqual(False, os.path.exists(f"{logPath}/taosdlog.0")) + tdSql.checkEqual(False, os.path.exists(f"{logPath}/taoslog.0")) + + def prepareCheckResources(self): + self.prepare_list = [["stdout", "10030"], ["stderr", "10031"], ["/dev/null", "10032"], + ["fullpath", "10033"], ["fullname", "10034"], ["relativepath", "10035"], + ["relativename", "10036"], ["filename", "10037"], ["empty", "10038"]] self.check_functions = { - "check_function_1": self.check_function_1, - "check_function_2": self.check_function_2, - "check_function_3": self.check_function_3 + self.prepare_stdout: self.check_stdout, + self.prepare_stderr: self.check_stderr, + self.prepare_dev_null: self.check_dev_null, + self.prepare_fullpath: self.check_fullpath, + self.prepare_fullname: self.check_fullname, + self.prepare_relativepath: self.check_relativepath, + self.prepare_relativename: self.check_relativename, + self.prepare_filename: self.check_filename, + self.prepare_empty: self.check_empty, } def checkLogOutput(self): - self.prepareCheckFunctions() - for key, check_func in self.check_functions.items(): - print(f"Running {key}") - check_func() + self.closeTaosd() + self.closeTaos() + self.prepareCheckResources() + with concurrent.futures.ThreadPoolExecutor() as executor: + prepare_futures = [executor.submit(prepare_func) for prepare_func, _ in self.check_functions.items()] + for future in concurrent.futures.as_completed(prepare_futures): + try: + future.result() + except Exception as e: + raise Exception(f"Error in prepare function: {e}") + + check_futures = [executor.submit(check_func) for _, check_func in self.check_functions.items()] + for future in concurrent.futures.as_completed(check_futures): + try: + future.result() + except Exception as e: + raise Exception(f"Error in prepare function: {e}") + self.closeTaosd() + self.closeTaos() + + def is_windows(self): + return os.name == 'nt' def run(self): - # self.logPathBasic() self.checkLogOutput() def stop(self):