test: add test case for log output

This commit is contained in:
kailixu 2024-12-03 07:03:53 +08:00
parent 09e1abc967
commit 1e8a2ab839
5 changed files with 292 additions and 56 deletions

View File

@ -1032,6 +1032,9 @@ static int32_t taosSetLogOutput(SConfig *pCfg) {
}
TAOS_CHECK_RETURN(cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true));
}
} else {
tstrncpy(tsLogDir, pLog, PATH_MAX);
TAOS_CHECK_RETURN(cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true));
}
}
return 0;

View File

@ -240,23 +240,28 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
}
} else if (strcmp(argv[i], "-k") == 0) {
global.generateGrant = true;
} else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--log-output") == 0) {
if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) {
printf("failed to set log output since length overflow, max length is %d\n", PATH_MAX);
} else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--log-output") == 0 ||
strncmp(argv[i], "--log-output=", 13) == 0) {
if ((i < argc - 1) || ((i == argc - 1) && strncmp(argv[i], "--log-output=", 13) == 0)) {
int32_t klen = strlen(argv[i]);
int32_t vlen = klen < 13 ? strlen(argv[++i]) : klen - 13;
const char *val = argv[i];
if (klen >= 13) val += 13;
if (vlen <= 0 || vlen >= PATH_MAX) {
printf("failed to set log output since invalid vlen:%d, valid range: [1, %d)\n", vlen, PATH_MAX);
return TSDB_CODE_INVALID_CFG;
}
tsLogOutput = taosMemoryMalloc(PATH_MAX);
if (!tsLogOutput) {
printf("failed to set log output: '%s' since %s\n", argv[i], tstrerror(terrno));
printf("failed to set log output: '%s' since %s\n", val, tstrerror(terrno));
return terrno;
}
if (taosExpandDir(argv[i], tsLogOutput, PATH_MAX) != 0) {
printf("failed to expand log output: '%s' since %s\n", argv[i], tstrerror(terrno));
if (taosExpandDir(val, tsLogOutput, PATH_MAX) != 0) {
printf("failed to expand log output: '%s' since %s\n", val, tstrerror(terrno));
return terrno;
}
} else {
printf("'-o' requires a parameter\n");
printf("'%s' requires a parameter\n", argv[i]);
return TSDB_CODE_INVALID_CFG;
}
} else if (strcmp(argv[i], "-y") == 0) {
@ -402,6 +407,7 @@ int mainWindows(int argc, char **argv) {
int32_t code = 0;
#endif
char *p = taosMemoryCalloc(10, 10);
if (global.generateGrant) {
dmGenerateGrant();
taosCleanupArgs();

View File

@ -1078,10 +1078,12 @@ static void taosWriteLog(SLogBuff *pLogBuf) {
}
#define LOG_ROTATE_INTERVAL 1800
#ifdef BUILD_TEST
#ifdef ASSERT_NOT_CORE
#define LOG_INACTIVE_TIME 3600
#define LOG_ROTATE_BOOT 5
#else
#define LOG_ROTATE_BOOT 180
#define LOG_INACTIVE_TIME 5
#define LOG_ROTATE_BOOT (LOG_INACTIVE_TIME + 1)
#endif
static void *taosLogRotateFunc(void *param) {
@ -1118,7 +1120,7 @@ static void *taosLogRotateFunc(void *param) {
}
int64_t fileSec = 0;
if ((code = taosStr2int64(pSec + 1, NULL)) != 0) {
if ((code = taosStr2int64(pSec + 1, &fileSec)) != 0) {
uWarn("%s:%d failed to convert %s to int64 since %s", __func__, __LINE__, pSec + 1, tstrerror(code));
continue;
}
@ -1132,19 +1134,19 @@ static void *taosLogRotateFunc(void *param) {
continue;
}
int64_t elapseSec = taosGetTimestampMs() / 1000 - mtime;
int64_t inactiveSec = taosGetTimestampMs() / 1000 - mtime;
if (elapseSec < 7200) {
if (inactiveSec < LOG_INACTIVE_TIME) {
continue;
}
char fullName[PATH_MAX] = {0};
snprintf(fullName, sizeof(fullName), "%s%s%s", tsLogDir, TD_DIRSEP, fname);
int32_t days = elapseSec / 86400 + 1;
int32_t days = inactiveSec / 86400 + 1;
if (tsLogKeepDays > 0 && days > tsLogKeepDays) {
TAOS_UNUSED(taosRemoveFile(fullName));
uInfo("file:%s is removed, days:%d keepDays:%d, sed:%" PRId64, fullName, days, tsLogKeepDays, fileSec);
uInfo("file:%s is removed, days:%d, keepDays:%d, sed:%" PRId64, fullName, days, tsLogKeepDays, fileSec);
} else {
taosKeepOldLog(fullName); // compress
}

View File

@ -352,6 +352,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/telemetry.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/backquote_check.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdMonitor.py
,,n,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdlog.py
,,n,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3
,,n,system-test,python3 ./test.py -f 0-others/udfTest.py
,,n,system-test,python3 ./test.py -f 0-others/udf_create.py

View File

@ -2,11 +2,18 @@ import taos
import sys
import time
import os
import os.path
import platform
import concurrent.futures
import shutil # Add this import
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
import time
import socket
import subprocess
class TDTestCase:
@ -20,8 +27,13 @@ class TDTestCase:
tdLog.exit("taosd not found!")
else:
tdLog.info("taosd found in %s" % self.buildPath)
self.taosdPath = self.buildPath + "/build/bin/taosd"
self.taosPath = self.buildPath + "/build/bin/taos"
self.logPath = self.buildPath + "/../sim/dnode1/log"
tdLog.info("taosd path: %s" % self.taosdPath)
tdLog.info("taos path: %s" % self.taosPath)
tdLog.info("log path: %s" % self.logPath)
self.commonCfgDict = {}
def getBuildPath(self):
selfPath = os.path.dirname(os.path.realpath(__file__))
@ -39,60 +51,272 @@ class TDTestCase:
break
return buildPath
def logPathBasic(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
tdSql.query("create user testpy pass 'testpy'")
tdDnodes.stop(1)
time.sleep(2)
tdSql.error("select * from information_schema.ins_databases")
tdSql.checkRows(2)
os.system("rm -rf %s" % self.logPath)
if os.path.exists(self.logPath) == True:
tdLog.exit("log path still exist!")
tdDnodes.start(1)
time.sleep(2)
if os.path.exists(self.logPath) != True:
tdLog.exit("log path is not generated!")
tdSql.query("select * from information_schema.ins_databases")
tdSql.checkRows(2)
def prepareCfg(self, cfgPath, cfgDict):
tdLog.info("make dir %s" % cfgPath)
os.makedirs(cfgPath, exist_ok=True)
with open(cfgPath + "/taos.cfg", "w") as f:
for key in self.commonCfgDict:
f.write("%s %s\n" % (key, self.commonCfgDict[key]))
for key in cfgDict:
f.write("%s %s\n" % (key, cfgDict[key]))
if not os.path.exists(cfgPath + "/taos.cfg"):
tdLog.exit("taos.cfg not found in %s" % cfgPath)
else:
tdLog.info("taos.cfg found in %s" % cfgPath)
def check_function_1(self):
# Implementation of check function 1
tdLog.info("Running check function 1")
# Add your check logic here
def checkProcessPid(self,processName):
i=0
while i<60:
tdLog.info(f"wait stop {processName}")
processPid = subprocess.getstatusoutput(f'ps aux|grep {processName} |grep -v "grep"|awk \'{{print $2}}\'')[1]
tdLog.info(f"times:{i},{processName}-pid:{processPid}")
if(processPid == ""):
break
i += 1
time.sleep(1)
else:
tdLog.info(f'this processName is not stoped in 60s')
def check_function_2(self):
# Implementation of check function 2
tdLog.info("Running check function 2")
# Add your check logic here
def closeTaosd(self, signal=9):
tdLog.info("Closing taosd")
if platform.system().lower() == 'windows':
psCmd = "for /f %%a in ('wmic process where \"name='taosd.exe' and CommandLine like '%%dnode%d%%'\" get processId ^| xargs echo ^| awk ^'{print $2}^' ^&^& echo aa') do @(ps | grep %%a | awk '{print $1}' | xargs)" % (self.index)
else:
psCmd = "ps -ef | grep -w taosd | grep -v grep | awk '{print $2}' | xargs"
tdLog.info(f"psCmd:{psCmd}")
def check_function_3(self):
# Implementation of check function 3
tdLog.info("Running check function 3")
# Add your check logic here
try:
rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
processID = rem.stdout.decode().strip()
tdLog.info(f"processID:{processID}")
except Exception as e:
tdLog.info(f"closeTaosd error:{e}")
processID = ""
tdLog.info(f"processID:{processID}")
onlyKillOnceWindows = 0
while(processID):
if not platform.system().lower() == 'windows' or (onlyKillOnceWindows == 0 and platform.system().lower() == 'windows'):
killCmd = "kill -%d %s > /dev/null 2>&1" % (signal, processID)
if platform.system().lower() == 'windows':
killCmd = "kill -%d %s > nul 2>&1" % (signal, processID)
tdLog.info(f"killCmd:{killCmd}")
os.system(killCmd)
tdLog.info(f"killed taosd process {processID}")
onlyKillOnceWindows = 1
time.sleep(1)
try:
rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
processID = rem.stdout.decode().strip()
except Exception as e:
tdLog.info(f"closeTaosd error:{e}")
processID = ""
def prepareCheckFunctions(self):
def closeTaos(self, signal=9):
tdLog.info("Closing taos")
if platform.system().lower() == 'windows':
psCmd = "for /f %%a in ('wmic process where \"name='taos.exe'\" get processId ^| xargs echo ^| awk ^'{print $2}^' ^&^& echo aa') do @(ps | grep %%a | awk '{print $1}' | xargs)"
else:
psCmd = "ps -ef | grep -w taos | grep -v grep | awk '{print $2}'"
tdLog.info(f"psCmd:{psCmd}")
try:
rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
processID = rem.stdout.decode().strip()
tdLog.info(f"processID:{processID}")
except Exception as e:
tdLog.info(f"closeTaos error:{e}")
processID = ""
tdLog.info(f"processID:{processID}")
onlyKillOnceWindows = 0
while(processID):
if not platform.system().lower() == 'windows' or (onlyKillOnceWindows == 0 and platform.system().lower() == 'windows'):
killCmd = "kill -%d %s > /dev/null 2>&1" % (signal, processID)
if platform.system().lower() == 'windows':
killCmd = "kill -%d %s > nul 2>&1" % (signal, processID)
tdLog.info(f"killCmd:{killCmd}")
os.system(killCmd)
tdLog.info(f"killed taos process {processID}")
onlyKillOnceWindows = 1
time.sleep(1)
try:
rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
processID = rem.stdout.decode().strip()
except Exception as e:
tdLog.info(f"closeTaos error:{e}")
processID = ""
def openBin(self, binPath, waitSec=5):
tdLog.info(f"Opening {binPath}")
try:
process = subprocess.Popen(binPath, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
time.sleep(waitSec)
if process.poll() is None:
tdLog.info(f"{binPath} started successfully")
else:
error = process.stderr.read().decode(encoding="utf-8").strip()
raise Exception(f"Failed to start {binPath}: {error}")
except Exception as e:
raise Exception(f"Failed to start {binPath}: %s" % repr(e))
def openTaosd(self, args="", waitSec=8):
self.openBin(f'{self.taosdPath} {args}', waitSec)
def openTaos(self, args="", waitSec=3):
self.openBin(f'{self.taosPath} {args}', waitSec)
def prepare_logoutput(self, desc, port, logOutput):
tdLog.info("Preparing %s, port:%s" % (desc, port))
dnodePath = self.buildPath + "/../sim/dnode%s" % port
tdLog.info("remove dnodePath:%s" % dnodePath)
try:
shutil.rmtree(dnodePath)
except Exception as e:
tdLog.info(f"Failed to remove directory {dnodePath}: {e}")
try:
self.prepareCfg(dnodePath, {"serverPort": port,
"dataDir": dnodePath + os.sep + "data",
"logDir": dnodePath + os.sep + "log"})
except Exception as e:
tdLog.info(f"Failed to prepare configuration for {dnodePath}: {e}")
return
try:
self.openTaosd(f"-c {dnodePath} {logOutput}")
self.openTaos(f"-c {dnodePath} {logOutput}")
except Exception as e:
tdLog.info(f"Failed to open taosd or taos for {dnodePath}: {e}")
def prepare_stdout(self):
list = self.prepare_list[0]
self.prepare_logoutput(list[0], list[1], "-o " + list[0])
def check_stdout(self):
tdLog.info("Running check stdout")
dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[0][1]
tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0"))
tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0"))
def prepare_stderr(self):
list = self.prepare_list[1]
self.prepare_logoutput(list[0], list[1], "--log-output " + list[0])
def check_stderr(self):
tdLog.info("Running check stderr")
dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[1][1]
tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0"))
tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0"))
def prepare_dev_null(self):
list = self.prepare_list[2]
self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0])
def check_dev_null(self):
tdLog.info("Running check /dev/null")
dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[2][1]
tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0"))
tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0"))
def prepare_fullpath(self):
list = self.prepare_list[3]
dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[3][1]
self.prepare_logoutput(list[0], list[1], "-o " + dnodePath + "/log0/" )
def check_fullpath(self):
tdLog.info("Running check fullpath")
logPath = self.buildPath + "/../sim/dnode%s/log0/" % self.prepare_list[3][1]
tdSql.checkEqual(True, os.path.exists(f"{logPath}taosdlog.0"))
tdSql.checkEqual(True, os.path.exists(f"{logPath}taoslog0.0"))
def prepare_fullname(self):
list = self.prepare_list[4]
dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[4][1]
self.prepare_logoutput(list[0], list[1], "--log-output " + dnodePath + "/log0/" + list[0])
dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[4][1]
def check_fullname(self):
tdLog.info("Running check fullname")
logPath = self.buildPath + "/../sim/dnode%s/log0/" % self.prepare_list[4][1]
tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[4][0] + ".0"))
tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[4][0] + "0.0"))
def prepare_relativepath(self):
list = self.prepare_list[5]
self.prepare_logoutput(list[0], list[1], "--log-output=" + "log0/")
def check_relativepath(self):
tdLog.info("Running check relativepath")
logPath = self.buildPath + "/../sim/dnode%s/log/log0/" % self.prepare_list[5][1]
tdSql.checkEqual(True, os.path.exists(logPath + "taosdlog.0"))
tdSql.checkEqual(True, os.path.exists(logPath + "taoslog0.0"))
def prepare_relativename(self):
list = self.prepare_list[6]
self.prepare_logoutput(list[0], list[1], "-o " + "log0/" + list[0])
def check_relativename(self):
tdLog.info("Running check relativename")
logPath = self.buildPath + "/../sim/dnode%s/log/log0/" % self.prepare_list[6][1]
tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[6][0] + ".0"))
tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[6][0] + "0.0"))
def prepare_filename(self):
list = self.prepare_list[7]
self.prepare_logoutput(list[0], list[1], "--log-output " + list[0])
def check_filename(self):
tdLog.info("Running check filename")
logPath = self.buildPath + "/../sim/dnode%s/log/" % self.prepare_list[7][1]
tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[7][0] + ".0"))
tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[7][0] + "0.0"))
def prepare_empty(self):
list = self.prepare_list[8]
self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0])
def check_empty(self):
tdLog.info("Running check empty")
logPath = self.buildPath + "/../sim/dnode%s/log" % self.prepare_list[8][1]
tdSql.checkEqual(False, os.path.exists(f"{logPath}/taosdlog.0"))
tdSql.checkEqual(False, os.path.exists(f"{logPath}/taoslog.0"))
def prepareCheckResources(self):
self.prepare_list = [["stdout", "10030"], ["stderr", "10031"], ["/dev/null", "10032"],
["fullpath", "10033"], ["fullname", "10034"], ["relativepath", "10035"],
["relativename", "10036"], ["filename", "10037"], ["empty", "10038"]]
self.check_functions = {
"check_function_1": self.check_function_1,
"check_function_2": self.check_function_2,
"check_function_3": self.check_function_3
self.prepare_stdout: self.check_stdout,
self.prepare_stderr: self.check_stderr,
self.prepare_dev_null: self.check_dev_null,
self.prepare_fullpath: self.check_fullpath,
self.prepare_fullname: self.check_fullname,
self.prepare_relativepath: self.check_relativepath,
self.prepare_relativename: self.check_relativename,
self.prepare_filename: self.check_filename,
self.prepare_empty: self.check_empty,
}
def checkLogOutput(self):
self.prepareCheckFunctions()
for key, check_func in self.check_functions.items():
print(f"Running {key}")
check_func()
self.closeTaosd()
self.closeTaos()
self.prepareCheckResources()
with concurrent.futures.ThreadPoolExecutor() as executor:
prepare_futures = [executor.submit(prepare_func) for prepare_func, _ in self.check_functions.items()]
for future in concurrent.futures.as_completed(prepare_futures):
try:
future.result()
except Exception as e:
raise Exception(f"Error in prepare function: {e}")
check_futures = [executor.submit(check_func) for _, check_func in self.check_functions.items()]
for future in concurrent.futures.as_completed(check_futures):
try:
future.result()
except Exception as e:
raise Exception(f"Error in prepare function: {e}")
self.closeTaosd()
self.closeTaos()
def is_windows(self):
return os.name == 'nt'
def run(self):
# self.logPathBasic()
self.checkLogOutput()
def stop(self):