From 74f8a252c629ec14f395607d59f893cf2ebbd2bf Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 8 Feb 2025 14:14:58 +0800 Subject: [PATCH 1/7] enh: solve conflict during process log file --- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/util/src/tlog.c | 24 ++++++++++++++++-------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 90f1bd9b8e..7c82b08e3c 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -205,7 +205,7 @@ typedef struct { typedef struct { int64_t id; - char name[TSDB_CLUSTER_ID_LEN]; + char nameX[TSDB_CLUSTER_ID_LEN]; int64_t createdTime; int64_t updateTime; int32_t upTime; diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 4f5ca8d789..0803040211 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -385,13 +385,21 @@ static void taosReserveOldLog(char *oldName, char *keepName) { static void taosKeepOldLog(char *oldName) { if (oldName[0] != 0) { - char compressFileName[PATH_MAX + 20]; - snprintf(compressFileName, PATH_MAX + 20, "%s.gz", oldName); - if (taosCompressFile(oldName, compressFileName) == 0) { - int32_t code = taosRemoveFile(oldName); - if (code != 0) { - TAOS_UNUSED(printf("failed to remove file:%s, reason:%s\n", oldName, tstrerror(code))); - } + int32_t code = 0, lino = 0; + TdFilePtr oldFile = NULL; + if ((oldFile = taosOpenFile(oldName, TD_FILE_READ))) { + TAOS_CHECK_GOTO(taosLockFile(oldFile), &lino, _exit2); + char compressFileName[PATH_MAX + 20]; + snprintf(compressFileName, PATH_MAX + 20, "%s.gz", oldName); + TAOS_CHECK_GOTO(taosCompressFile(oldName, compressFileName), &lino, _exit1); + TAOS_CHECK_GOTO(taosRemoveFile(oldName), &lino, _exit1); + } + _exit1: + TAOS_UNUSED(taosUnLockFile(oldFile)); + _exit2: + TAOS_UNUSED(taosCloseFile(&oldFile)); + if (code != 0) { + uWarn("failed at line %d to keep old log file:%s, reason:%s\n", oldName, lino, tstrerror(code)); } } } @@ -1041,7 +1049,7 @@ static void taosWriteLog(SLogBuff *pLogBuf) { } #define LOG_ROTATE_INTERVAL 3600 -#if !defined(TD_ENTERPRISE) || defined(ASSERT_NOT_CORE) +#if !defined(TD_ENTERPRISE) || defined(ASSERT_NOT_CORE) || defined(GRANTS_CFG) #define LOG_INACTIVE_TIME 7200 #define LOG_ROTATE_BOOT 900 #else From 34896f15e44e91bfaf8c6e1ab3e8756a6a201b71 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 8 Feb 2025 14:17:00 +0800 Subject: [PATCH 2/7] enh: solve conflict during process log file --- source/dnode/mnode/impl/inc/mndDef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 7c82b08e3c..90f1bd9b8e 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -205,7 +205,7 @@ typedef struct { typedef struct { int64_t id; - char nameX[TSDB_CLUSTER_ID_LEN]; + char name[TSDB_CLUSTER_ID_LEN]; int64_t createdTime; int64_t updateTime; int32_t upTime; From fea6148c968b8d36b9a049639feeb9c7854858f8 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 8 Feb 2025 14:19:06 +0800 Subject: [PATCH 3/7] enh: solve conflict during process log file --- source/util/src/tlog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 0803040211..eb4b4de863 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -399,7 +399,7 @@ static void taosKeepOldLog(char *oldName) { _exit2: TAOS_UNUSED(taosCloseFile(&oldFile)); if (code != 0) { - uWarn("failed at line %d to keep old log file:%s, reason:%s\n", oldName, lino, tstrerror(code)); + uWarn("failed at line %d to keep old log file:%s, reason:%s\n", lino, oldName, tstrerror(code)); } } } From f9dc305e3583771739a5370fd8c1ec3715c6f9fd Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 8 Feb 2025 16:08:40 +0800 Subject: [PATCH 4/7] enh: print err msg only in embedded log mode --- source/util/src/tlog.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index eb4b4de863..94b6b67018 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -398,8 +398,9 @@ static void taosKeepOldLog(char *oldName) { TAOS_UNUSED(taosUnLockFile(oldFile)); _exit2: TAOS_UNUSED(taosCloseFile(&oldFile)); - if (code != 0) { - uWarn("failed at line %d to keep old log file:%s, reason:%s\n", lino, oldName, tstrerror(code)); + if (code != 0 && tsLogEmbedded == 1) { // only print error message in embedded log mode + // don't use uWarn or uError, because it may open new log file and cause dead lock + fprintf(stderr, "failed at line %d to keep old log file:%s, reason:%s\n", lino, oldName, tstrerror(code)); } } } From 3edb8ae41c0565199f483a3a8876c182ed1091a6 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 9 Feb 2025 09:31:41 +0800 Subject: [PATCH 5/7] chore: logic optimization --- source/util/src/tlog.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 94b6b67018..8537f35095 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -393,13 +393,15 @@ static void taosKeepOldLog(char *oldName) { snprintf(compressFileName, PATH_MAX + 20, "%s.gz", oldName); TAOS_CHECK_GOTO(taosCompressFile(oldName, compressFileName), &lino, _exit1); TAOS_CHECK_GOTO(taosRemoveFile(oldName), &lino, _exit1); + _exit1: + TAOS_UNUSED(taosUnLockFile(oldFile)); + _exit2: + TAOS_UNUSED(taosCloseFile(&oldFile)); + } else { + code = terrno; } - _exit1: - TAOS_UNUSED(taosUnLockFile(oldFile)); - _exit2: - TAOS_UNUSED(taosCloseFile(&oldFile)); - if (code != 0 && tsLogEmbedded == 1) { // only print error message in embedded log mode - // don't use uWarn or uError, because it may open new log file and cause dead lock + if (code != 0 && tsLogEmbedded == 1) { // print error messages only in embedded log mode + // avoid using uWarn or uError, as they may open a new log file and potentially cause a deadlock. fprintf(stderr, "failed at line %d to keep old log file:%s, reason:%s\n", lino, oldName, tstrerror(code)); } } From 0e502e05e760194d1eb75ac6aaf7382b2acc0c0e Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 11 Feb 2025 20:03:11 +0800 Subject: [PATCH 6/7] test: add test case for log compress --- tests/system-test/0-others/taosdlog.py | 73 ++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/tests/system-test/0-others/taosdlog.py b/tests/system-test/0-others/taosdlog.py index 06ac4ce5e4..7e8dc49552 100644 --- a/tests/system-test/0-others/taosdlog.py +++ b/tests/system-test/0-others/taosdlog.py @@ -47,6 +47,78 @@ class TDTestCase: break return buildPath + def checkLogBak(self, logPath, expectLogBak): + if platform.system().lower() == 'windows': + return True + result = False + try: + for file in os.listdir(logPath): + file_path = os.path.join(logPath, file) + if os.path.isdir(file_path): + continue + if file.endswith('.gz'): + if expectLogBak: + result = True + else: + raise Exception(f"Error: Found .gz file: {file_path}") + if '.' in file: + prefix, num_part = file.split('.', 1) + logNum=0 + if num_part.isdigit(): + logNum = int(num_part) + if logNum > 100: + if not expectLogBak: + raise Exception(f"Error: Found log file number >= 100: {file_path}") + except Exception as e: + raise Exception(f"Error: error occurred. Reason: {e}") + return result + + def checkTargetStrInFiles(self, filePaths, targetStr): + result = False + for filePath in filePaths: + if not os.path.exists(filePath): + continue + try: + with open(filePath, 'r', encoding='utf-8') as file: + for line in file: + if targetStr in line: + result = True + break + except Exception as e: + continue + return result + + def logRotateOccurred(self, logFiles, targetStr, maxRetry=15): + result = False + for i in range(maxRetry): + if self.checkTargetStrInFiles(logFiles, targetStr): + result = True + break + tdLog.info(f"wait {i+1} second(s) for log rotate") + time.sleep(1) + return result + + def checkLogCompress(self): + tdLog.info("Running check log compress") + dnodePath = self.buildPath + "/../sim/dnode1" + logPath = f"{dnodePath}/log" + taosdLogFiles = [f"{logPath}/taosdlog.0", f"{logPath}/taosdlog.1"] + logRotateStr="process log rotation" + logRotateResult = self.logRotateOccurred(taosdLogFiles, logRotateStr) + tdSql.checkEqual(True, logRotateResult) + tdSql.checkEqual(False, self.checkLogBak(logPath, False)) + tdSql.execute("alter all dnodes 'logKeepDays 3'") + tdSql.execute("alter all dnodes 'numOfLogLines 1000'") + tdSql.execute("alter all dnodes 'debugFlag 143'") + logCompress=False + for i in range(30): + logCompress=self.checkLogBak(logPath, True) + if logCompress: + break + tdLog.info(f"wait {i+1} second(s) for log compress") + time.sleep(1) + tdSql.checkEqual(True, logCompress) + def prepareCfg(self, cfgPath, cfgDict): tdLog.info("make dir %s" % cfgPath) os.makedirs(cfgPath, exist_ok=True) @@ -338,6 +410,7 @@ class TDTestCase: tdSql.checkEqual(True, os.path.exists(f"{dnodePath}/log/taoslog0.0")) def run(self): + self.checkLogCompress() self.checkLogOutput() self.checkLogRotate() self.closeTaosd() From 1c9b7d59b00b4311b68bee9cee96fe519c37f3b0 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 12 Feb 2025 10:05:32 +0800 Subject: [PATCH 7/7] chore: prompt and test case for log --- source/util/src/tlog.c | 2 +- tests/system-test/0-others/taosdlog.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 8537f35095..a0105c260b 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -402,7 +402,7 @@ static void taosKeepOldLog(char *oldName) { } if (code != 0 && tsLogEmbedded == 1) { // print error messages only in embedded log mode // avoid using uWarn or uError, as they may open a new log file and potentially cause a deadlock. - fprintf(stderr, "failed at line %d to keep old log file:%s, reason:%s\n", lino, oldName, tstrerror(code)); + fprintf(stderr, "WARN: failed at line %d to keep old log file:%s, reason:%s\n", lino, oldName, tstrerror(code)); } } } diff --git a/tests/system-test/0-others/taosdlog.py b/tests/system-test/0-others/taosdlog.py index 7e8dc49552..b6efbe2a4d 100644 --- a/tests/system-test/0-others/taosdlog.py +++ b/tests/system-test/0-others/taosdlog.py @@ -118,6 +118,8 @@ class TDTestCase: tdLog.info(f"wait {i+1} second(s) for log compress") time.sleep(1) tdSql.checkEqual(True, logCompress) + tdSql.execute("alter all dnodes 'numOfLogLines 1000000'") + tdSql.execute("alter all dnodes 'debugFlag 131'") def prepareCfg(self, cfgPath, cfgDict): tdLog.info("make dir %s" % cfgPath)