From 4d19f48d8a00c14ebe270f50bcfff550a9cd06b5 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 2 Aug 2024 16:29:48 +0800 Subject: [PATCH 1/8] feat:[TS-3718]save slow log file one day one file --- include/util/tlog.h | 2 +- source/common/src/tglobal.c | 2 +- source/dnode/mgmt/test/sut/src/sut.cpp | 2 +- source/dnode/mnode/impl/test/sdb/sdbTest.cpp | 2 +- source/dnode/mnode/impl/test/trans/trans2.cpp | 2 +- source/libs/executor/test/joinTests.cpp | 2 +- source/libs/index/test/fstUT.cc | 2 +- source/libs/index/test/indexBench.cc | 2 +- source/libs/index/test/indexTests.cc | 2 +- .../libs/index/test/index_executor_tests.cpp | 2 +- source/libs/index/test/jsonUT.cc | 2 +- source/libs/parser/test/parTestMain.cpp | 2 +- source/libs/planner/test/planTestMain.cpp | 2 +- source/libs/qworker/test/qworkerTests.cpp | 2 +- .../libs/scalar/test/filter/filterTests.cpp | 2 +- .../libs/scalar/test/scalar/scalarTests.cpp | 2 +- source/libs/scheduler/test/schedulerTests.cpp | 2 +- source/libs/sync/test/syncTest.cpp | 2 +- source/libs/sync/test/syncTestTool.cpp | 2 +- source/libs/transport/test/cliBench.c | 2 +- source/libs/transport/test/http_test.c | 2 +- source/libs/transport/test/svrBench.c | 2 +- source/libs/transport/test/transUT.cpp | 2 +- source/util/src/tlog.c | 239 +++++++++++------- source/util/test/trefTest.c | 2 +- 25 files changed, 167 insertions(+), 120 deletions(-) diff --git a/include/util/tlog.h b/include/util/tlog.h index 67aafbfe44..c24a456c99 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -69,7 +69,7 @@ extern int32_t tdbDebugFlag; extern int32_t sndDebugFlag; extern int32_t simDebugFlag; -int32_t taosInitLog(const char *logName, int32_t maxFiles); +int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc); void taosCloseLog(); void taosResetLog(); void taosDumpData(uint8_t *msg, int32_t len); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 46219fe34c..b15ceaee7e 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1596,7 +1596,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi goto _exit; } - if ((code = taosInitLog(logname, logFileNum)) != 0) { + if ((code = taosInitLog(logname, logFileNum, tsc)) != 0) { printf("failed to init log file since %s\n", tstrerror(code)); goto _exit; } diff --git a/source/dnode/mgmt/test/sut/src/sut.cpp b/source/dnode/mgmt/test/sut/src/sut.cpp index b54590ce82..f074a015d2 100644 --- a/source/dnode/mgmt/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/test/sut/src/sut.cpp @@ -37,7 +37,7 @@ void Testbase::InitLog(const char* path) { taosGetSystemInfo(); tsRpcQueueMemoryAllowed = tsTotalMemoryKB * 0.1; - if (taosInitLog("taosdlog", 1) != 0) { + if (taosInitLog("taosdlog", 1, false) != 0) { printf("failed to init log file\n"); } } diff --git a/source/dnode/mnode/impl/test/sdb/sdbTest.cpp b/source/dnode/mnode/impl/test/sdb/sdbTest.cpp index b12ea9a28f..97b7ccd5f7 100644 --- a/source/dnode/mnode/impl/test/sdb/sdbTest.cpp +++ b/source/dnode/mnode/impl/test/sdb/sdbTest.cpp @@ -35,7 +35,7 @@ class MndTestSdb : public ::testing::Test { taosRemoveDir(path); taosMkDir(path); tstrncpy(tsLogDir, path, PATH_MAX); - if (taosInitLog("taosdlog", 1) != 0) { + if (taosInitLog("taosdlog", 1, false) != 0) { printf("failed to init log file\n"); } } diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index 4d0d53ced0..b73f07c778 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -52,7 +52,7 @@ class MndTestTrans2 : public ::testing::Test { taosRemoveDir(logpath); taosMkDir(logpath); tstrncpy(tsLogDir, logpath, PATH_MAX); - if (taosInitLog("taosdlog", 1) != 0) { + if (taosInitLog("taosdlog", 1, false) != 0) { printf("failed to init log file\n"); } } diff --git a/source/libs/executor/test/joinTests.cpp b/source/libs/executor/test/joinTests.cpp index 6100b0722d..7d3cf0580c 100755 --- a/source/libs/executor/test/joinTests.cpp +++ b/source/libs/executor/test/joinTests.cpp @@ -2884,7 +2884,7 @@ void jtInitLogFile() { qDebugFlag = 159; TAOS_STRCPY(tsLogDir, TD_LOG_DIR_PATH); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { JT_PRINTF("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/index/test/fstUT.cc b/source/libs/index/test/fstUT.cc index 37c9d1b97b..04bc6c9a50 100644 --- a/source/libs/index/test/fstUT.cc +++ b/source/libs/index/test/fstUT.cc @@ -28,7 +28,7 @@ static void EnvInit() { taosMkDir(path.c_str()); // init log file tstrncpy(tsLogDir, path.c_str(), PATH_MAX); - if (taosInitLog("tindex.idx", 1) != 0) { + if (taosInitLog("tindex.idx", 1, false) != 0) { printf("failed to init log"); } // init index file diff --git a/source/libs/index/test/indexBench.cc b/source/libs/index/test/indexBench.cc index 78c687029c..369ed54d0b 100644 --- a/source/libs/index/test/indexBench.cc +++ b/source/libs/index/test/indexBench.cc @@ -40,7 +40,7 @@ static void initLog() { taosRemoveDir(tsLogDir); taosMkDir(tsLogDir); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 1c5a04d22d..090ff02ffb 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -283,7 +283,7 @@ static void initLog() { taosRemoveDir(tsLogDir); taosMkDir(tsLogDir); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/index/test/index_executor_tests.cpp b/source/libs/index/test/index_executor_tests.cpp index 232675d32f..f65028e564 100644 --- a/source/libs/index/test/index_executor_tests.cpp +++ b/source/libs/index/test/index_executor_tests.cpp @@ -57,7 +57,7 @@ void sifInitLogFile() { taosRemoveDir(tsLogDir); taosMkDir(tsLogDir); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index bee8e0dd85..493101eb27 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -40,7 +40,7 @@ static void initLog() { taosRemoveDir(tsLogDir); taosMkDir(tsLogDir); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/parser/test/parTestMain.cpp b/source/libs/parser/test/parTestMain.cpp index 87ecf34ebc..3986c96f07 100644 --- a/source/libs/parser/test/parTestMain.cpp +++ b/source/libs/parser/test/parTestMain.cpp @@ -72,7 +72,7 @@ class ParserEnv : public testing::Environment { taosRemoveDir(path); ASSERT_EQ(TSDB_CODE_SUCCESS, taosMkDir(path)); tstrncpy(tsLogDir, path, PATH_MAX); - if (taosInitLog("taoslog", 1) != 0) { + if (taosInitLog("taoslog", 1, false) != 0) { std::cout << "failed to init log file" << std::endl; } } diff --git a/source/libs/planner/test/planTestMain.cpp b/source/libs/planner/test/planTestMain.cpp index 99f71e1bc6..2a1e169d5f 100644 --- a/source/libs/planner/test/planTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -67,7 +67,7 @@ class PlannerEnv : public testing::Environment { taosRemoveDir(path); ASSERT_EQ(TSDB_CODE_SUCCESS,taosMkDir(path)); tstrncpy(tsLogDir, path, PATH_MAX); - if (taosInitLog("taoslog", 1) != 0) { + if (taosInitLog("taoslog", 1, false) != 0) { std::cout << "failed to init log file" << std::endl; } } diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 91e1dda2ac..42b538b14f 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -105,7 +105,7 @@ void qwtInitLogFile() { qDebugFlag = 159; strcpy(tsLogDir, TD_LOG_DIR_PATH); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 80e2c005a7..33d2c1e2ef 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -57,7 +57,7 @@ void flttInitLogFile() { qDebugFlag = 159; (void)strcpy(tsLogDir, TD_LOG_DIR_PATH); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 75180c0b66..032b197046 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -83,7 +83,7 @@ void scltInitLogFile() { qDebugFlag = 159; (void)strcpy(tsLogDir, TD_LOG_DIR_PATH); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { (void)printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index f7c3fb8847..44d32b9480 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -79,7 +79,7 @@ void schtInitLogFile() { qDebugFlag = 159; TAOS_STRCPY(tsLogDir, TD_LOG_DIR_PATH); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { (void)printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index 7b636085f2..3a9a29f023 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -48,7 +48,7 @@ void test4() { } int main(int argc, char** argv) { - taosInitLog("/tmp/syncTest.log", 100); + taosInitLog("/tmp/syncTest.log", 100, false); tsAsyncLog = 0; sDebugFlag = DEBUG_SCREEN + DEBUG_FILE + DEBUG_TRACE + DEBUG_INFO + DEBUG_ERROR; diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp index 4bc2e92d0c..6671cebcaf 100644 --- a/source/libs/sync/test/syncTestTool.cpp +++ b/source/libs/sync/test/syncTestTool.cpp @@ -351,7 +351,7 @@ int main(int argc, char** argv) { char logFile[256]; snprintf(logFile, sizeof(logFile), "/tmp/%s-replicaNum%d-myIndex%d.log", gDir, replicaNum, myIndex); - taosInitLog(logFile, 100); + taosInitLog(logFile, 100, false); sTrace("logFile : %s", logFile); gSnapshotLastApplyIndex = lastApplyIndex; diff --git a/source/libs/transport/test/cliBench.c b/source/libs/transport/test/cliBench.c index 8a5276b814..2fb71062a2 100644 --- a/source/libs/transport/test/cliBench.c +++ b/source/libs/transport/test/cliBench.c @@ -43,7 +43,7 @@ void initLogEnv() { taosRemoveDir(tsLogDir); taosMkDir(tsLogDir); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/transport/test/http_test.c b/source/libs/transport/test/http_test.c index d04fb84843..1d0f0fa47a 100644 --- a/source/libs/transport/test/http_test.c +++ b/source/libs/transport/test/http_test.c @@ -31,7 +31,7 @@ void initLogEnv() { taosRemoveDir(tsLogDir); taosMkDir(tsLogDir); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/transport/test/svrBench.c b/source/libs/transport/test/svrBench.c index dff3efef12..6408e4dcb2 100644 --- a/source/libs/transport/test/svrBench.c +++ b/source/libs/transport/test/svrBench.c @@ -53,7 +53,7 @@ void initLogEnv() { taosRemoveDir(tsLogDir); taosMkDir(tsLogDir); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/transport/test/transUT.cpp b/source/libs/transport/test/transUT.cpp index 2fa94c358f..e57d01bcbc 100644 --- a/source/libs/transport/test/transUT.cpp +++ b/source/libs/transport/test/transUT.cpp @@ -230,7 +230,7 @@ static void initEnv() { taosMkDir(path.c_str()); tstrncpy(tsLogDir, path.c_str(), PATH_MAX); - if (taosInitLog("taosdlog", 1) != 0) { + if (taosInitLog("taosdlog", 1, false) != 0) { printf("failed to init log file\n"); } } diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 1946a0a274..0f217c439f 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -67,9 +67,12 @@ typedef struct { int32_t lines; int32_t flag; int32_t openInProgress; + int32_t openInProgressSlowLog; int64_t lastKeepFileSec; + char slowLogDay[64]; pid_t pid; char logName[LOG_FILE_NAME_LEN]; + char slowLogName[LOG_FILE_NAME_LEN]; SLogBuff *logHandle; SLogBuff *slowHandle; TdThreadMutex logMutex; @@ -77,7 +80,7 @@ typedef struct { extern SConfig *tsCfg; static int8_t tsLogInited = 0; -static SLogObj tsLogObj = {.fileNum = 1}; +static SLogObj tsLogObj = {.fileNum = 1, .slowHandle = NULL}; static int64_t tsAsyncLogLostLines = 0; static int32_t tsDaylightActive; /* Currently in daylight saving time. */ @@ -131,18 +134,8 @@ static void *taosAsyncOutputLog(void *param); static int32_t taosPushLogBuffer(SLogBuff *pLogBuf, const char *msg, int32_t msgLen); static SLogBuff *taosLogBuffNew(int32_t bufSize); static void taosCloseLogByFd(TdFilePtr pFile); -static int32_t taosOpenLogFile(char *fn, int32_t maxFileNum); - -static FORCE_INLINE void taosUpdateDaylight() { - struct tm Tm, *ptm; - struct timeval timeSecs; - (void)taosGetTimeOfDay(&timeSecs); - time_t curTime = timeSecs.tv_sec; - ptm = taosLocalTime(&curTime, &Tm, NULL); - tsDaylightActive = ptm->tm_isdst; -} -static FORCE_INLINE int32_t taosGetDaylight() { return tsDaylightActive; } - +static int32_t taosInitNormalLog(const char *fn, int32_t maxFileNum); +static void backUpSlowLog(); static int32_t taosStartLog() { TdThreadAttr threadAttr; (void)taosThreadAttrInit(&threadAttr); @@ -153,47 +146,7 @@ static int32_t taosStartLog() { return 0; } -int32_t taosInitSlowLog() { - char fullName[PATH_MAX] = {0}; - char logFileName[64] = {0}; -#ifdef CUS_PROMPT - snprintf(logFileName, 64, "%sSlowLog", CUS_PROMPT); -#else - snprintf(logFileName, 64, "taosSlowLog"); -#endif - - if (strlen(tsLogDir) != 0) { - char lastC = tsLogDir[strlen(tsLogDir) - 1]; - if (lastC == '\\' || lastC == '/') { - snprintf(fullName, PATH_MAX, - "%s" - "%s", - tsLogDir, logFileName); - } else { - snprintf(fullName, PATH_MAX, "%s" TD_DIRSEP "%s", tsLogDir, logFileName); - } - } else { - snprintf(fullName, PATH_MAX, "%s", logFileName); - } - - tsLogObj.slowHandle = taosLogBuffNew(LOG_SLOW_BUF_SIZE); - if (tsLogObj.slowHandle == NULL) return terrno; - - (void)taosUmaskFile(0); - tsLogObj.slowHandle->pFile = taosOpenFile(fullName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); - if (tsLogObj.slowHandle->pFile == NULL) { - printf("\nfailed to open slow log file:%s, reason:%s\n", fullName, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); - } - - return 0; -} - -int32_t taosInitLog(const char *logName, int32_t maxFiles) { - if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0; - osUpdate(); - - char fullName[PATH_MAX] = {0}; +static void getFullPathName(char* fullName, const char* logName){ if (strlen(tsLogDir) != 0) { char lastC = tsLogDir[strlen(tsLogDir) - 1]; if (lastC == '\\' || lastC == '/') { @@ -207,13 +160,40 @@ int32_t taosInitLog(const char *logName, int32_t maxFiles) { } else { snprintf(fullName, PATH_MAX, "%s", logName); } - taosUpdateDaylight(); +} - tsLogObj.logHandle = taosLogBuffNew(LOG_DEFAULT_BUF_SIZE); - if (tsLogObj.logHandle == NULL) return terrno; - TAOS_CHECK_RETURN(taosOpenLogFile(fullName, maxFiles)); +int32_t taosInitSlowLog() { + char logFileName[64] = {0}; +#ifdef CUS_PROMPT + snprintf(logFileName, 64, "%sSlowLog", CUS_PROMPT); +#else + snprintf(logFileName, 64, "taosSlowLog"); +#endif - TAOS_CHECK_RETURN(taosInitSlowLog()); + getFullPathName(tsLogObj.slowLogName, logFileName); + + tsLogObj.slowHandle = taosLogBuffNew(LOG_SLOW_BUF_SIZE); + if (tsLogObj.slowHandle == NULL) return terrno; + + (void)taosUmaskFile(0); + tsLogObj.slowHandle->pFile = taosOpenFile(tsLogObj.slowLogName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (tsLogObj.slowHandle->pFile == NULL) { + printf("\nfailed to open slow log file:%s, reason:%s\n", tsLogObj.slowLogName, strerror(errno)); + return TAOS_SYSTEM_ERROR(errno); + } + + backUpSlowLog(); + return 0; +} + +int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) { + if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0; + osUpdate(); + + TAOS_CHECK_RETURN(taosInitNormalLog(logName, maxFiles)); + if (tsc){ + TAOS_CHECK_RETURN(taosInitSlowLog()); + } TAOS_CHECK_RETURN(taosStartLog()); return 0; } @@ -383,6 +363,49 @@ static int32_t taosOpenNewLogFile() { return 0; } +static void taosOpenNewSlowLogFile(char* day) { + int32_t code = 0; + TdFilePtr pOldFile = tsLogObj.slowHandle->pFile; + if (taosLockFile(pOldFile) != 0){ + return; + } + + TdFilePtr pFile = NULL; + char name[LOG_FILE_NAME_LEN + 64] = {0}; + (void)sprintf(name, "%s.%s", tsLogObj.slowLogName, day); + if (taosCheckExistFile(name)){ + pFile = taosOpenFile(tsLogObj.slowLogName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pFile == NULL) { + uError("open new log file fail! reason:%s, reuse lastlog", strerror(errno)); + goto END; + } + }else{ + if ((code = taosRenameFile(tsLogObj.slowLogName, name))) { + uError("failed to rename file:%s to %s since %s", tsLogObj.slowLogName, name, tstrerror(code)); + goto END; + } + + (void)taosUmaskFile(0); + + pFile = taosOpenFile(tsLogObj.slowLogName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + uError("open new log file fail! reason:%s, reuse lastlog", strerror(errno)); + goto END; + } + (void)taosLSeekFile(pFile, 0, SEEK_SET); + } + + tsLogObj.slowHandle->pFile = pFile; + taosUnLockLogFile(pOldFile); + (void)taosCloseFile(&pOldFile); + tstrncpy(tsLogObj.slowLogDay, day, 64); + return; + +END: + taosUnLockLogFile(pOldFile); + +} + void taosResetLog() { // force create a new log file tsLogObj.lines = tsNumOfLogLines + 10; @@ -415,7 +438,8 @@ static bool taosCheckFileIsOpen(char *logFileName) { } } -static void taosGetLogFileName(char *fn) { +static void decideLogFileName(const char *fn, int32_t maxFileNum) { + tsLogObj.fileNum = maxFileNum; if (tsLogObj.fileNum > 1) { for (int32_t i = 0; i < tsLogObj.fileNum; i++) { char fileName[LOG_FILE_NAME_LEN]; @@ -438,32 +462,20 @@ static void taosGetLogFileName(char *fn) { } } -static int32_t taosOpenLogFile(char *fn, int32_t maxFileNum) { -#ifdef WINDOWS_STASH - /* - * always set maxFileNum to 1 - * means client log filename is unique in windows - */ - maxFileNum = 1; -#endif - +static void decideLogFileNameFlag(){ char name[LOG_FILE_NAME_LEN + 50] = "\0"; - int32_t logstat0_mtime, logstat1_mtime; + int32_t logstat0_mtime = 0; + int32_t logstat1_mtime = 0; + bool log0Exist = false; + bool log1Exist = false; - tsLogObj.fileNum = maxFileNum; - taosGetLogFileName(fn); - - if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) { - strcpy(name, fn); + if (strlen(tsLogObj.logName) < LOG_FILE_NAME_LEN + 50 - 2) { + strcpy(name, tsLogObj.logName); strcat(name, ".0"); + log0Exist = taosStatFile(name, NULL, &logstat0_mtime, NULL) >= 0; + name[strlen(name) - 1] = '1'; + log1Exist = taosStatFile(name, NULL, &logstat1_mtime, NULL) >= 0; } - bool log0Exist = taosStatFile(name, NULL, &logstat0_mtime, NULL) >= 0; - - if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) { - strcpy(name, fn); - strcat(name, ".1"); - } - bool log1Exist = taosStatFile(name, NULL, &logstat1_mtime, NULL) >= 0; // if none of the log files exist, open 0, if both exists, open the old one if (!log0Exist && !log1Exist) { @@ -475,16 +487,37 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxFileNum) { } else { tsLogObj.flag = (logstat0_mtime > logstat1_mtime) ? 0 : 1; } +} - char fileName[LOG_FILE_NAME_LEN + 50] = "\0"; - sprintf(fileName, "%s.%d", tsLogObj.logName, tsLogObj.flag); +static void processLogFileName(const char* logName , int32_t maxFileNum){ + char fullName[PATH_MAX] = {0}; + getFullPathName(fullName, logName); + decideLogFileName(fullName, maxFileNum); + decideLogFileNameFlag(); +} + +static int32_t taosInitNormalLog(const char *logName, int32_t maxFileNum) { +#ifdef WINDOWS_STASH + /* + * always set maxFileNum to 1 + * means client log filename is unique in windows + */ + maxFileNum = 1; +#endif + + processLogFileName(logName, maxFileNum); + + char name[LOG_FILE_NAME_LEN + 50] = "\0"; + sprintf(name, "%s.%d", tsLogObj.logName, tsLogObj.flag); (void)taosThreadMutexInit(&tsLogObj.logMutex, NULL); (void)taosUmaskFile(0); - tsLogObj.logHandle->pFile = taosOpenFile(fileName, TD_FILE_CREATE | TD_FILE_WRITE); + tsLogObj.logHandle = taosLogBuffNew(LOG_DEFAULT_BUF_SIZE); + if (tsLogObj.logHandle == NULL) return terrno; + tsLogObj.logHandle->pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE); if (tsLogObj.logHandle->pFile == NULL) { - printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno)); + printf("\nfailed to open log file:%s, reason:%s\n", name, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } (void)taosLockLogFile(tsLogObj.logHandle->pFile); @@ -492,7 +525,7 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxFileNum) { // only an estimate for number of lines int64_t filesize = 0; if (taosFStatFile(tsLogObj.logHandle->pFile, &filesize, NULL) < 0) { - printf("\nfailed to fstat log file:%s, reason:%s\n", fileName, strerror(errno)); + printf("\nfailed to fstat log file:%s, reason:%s\n", name, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } tsLogObj.lines = (int32_t)(filesize / 60); @@ -610,6 +643,21 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons taosMemoryFree(buffer); } +static void backUpSlowLog(){ + char buf[64] = {0}; + time_t t = taosTime(NULL); + struct tm tmInfo; + if (taosLocalTime(&t, &tmInfo, buf) != NULL) { + (void)strftime(buf, sizeof(buf), "%Y-%m-%d", &tmInfo); + } +// (void)taosThreadMutexLock(&tsLogObj.logMutex); +// if (strlen(tsLogObj.slowLogDay) == 0) { +// tstrncpy(tsLogObj.slowLogDay, buf, 64); +// }else if (strcmp(tsLogObj.slowLogDay, buf) != 0) { + (void)taosOpenNewSlowLogFile(buf); +// } + (void)taosThreadMutexUnlock(&tsLogObj.logMutex); +} void taosPrintSlowLog(const char *format, ...) { if (!osLogSpaceAvailable()) return; @@ -834,7 +882,11 @@ static void *taosAsyncOutputLog(void *param) { int32_t writeInterval = 0; while (1) { - writeInterval = TMIN(pLogBuf->writeInterval, pSlowBuf->writeInterval); + if (pSlowBuf) { + writeInterval = TMIN(pLogBuf->writeInterval, pSlowBuf->writeInterval); + } else { + writeInterval = pLogBuf->writeInterval; + } count += writeInterval; updateCron++; taosMsleep(writeInterval); @@ -845,17 +897,12 @@ static void *taosAsyncOutputLog(void *param) { // Polling the buffer taosWriteLog(pLogBuf); - taosWriteLog(pSlowBuf); + if (pSlowBuf) taosWriteLog(pSlowBuf); - if (updateCron >= 3600 * 24 * 40 / 2) { - taosUpdateDaylight(); - updateCron = 0; - } - - if (pLogBuf->stop || pSlowBuf->stop) { + if (pLogBuf->stop || (pSlowBuf && pSlowBuf->stop)) { pLogBuf->lastDuration = LOG_MAX_WAIT_MSEC; taosWriteLog(pLogBuf); - taosWriteLog(pSlowBuf); + if (pSlowBuf) taosWriteLog(pSlowBuf); break; } } diff --git a/source/util/test/trefTest.c b/source/util/test/trefTest.c index 46656d7776..7abdef7c78 100644 --- a/source/util/test/trefTest.c +++ b/source/util/test/trefTest.c @@ -154,7 +154,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("tref.log", 10); + taosInitLog("tref.log", 10, false); SRefSpace *pSpaceList = (SRefSpace *)taosMemoryCalloc(sizeof(SRefSpace), threads); TdThread *pThreadList = (TdThread *)taosMemoryCalloc(sizeof(TdThread), threads); From f5e35549d714d1810a15a86985a148b9ac95b644 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 2 Aug 2024 17:10:21 +0800 Subject: [PATCH 2/8] feat:[TS-3718]save slow log file one day one file --- source/util/src/tlog.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 0f217c439f..199e030d82 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -182,7 +182,6 @@ int32_t taosInitSlowLog() { return TAOS_SYSTEM_ERROR(errno); } - backUpSlowLog(); return 0; } @@ -403,7 +402,6 @@ static void taosOpenNewSlowLogFile(char* day) { END: taosUnLockLogFile(pOldFile); - } void taosResetLog() { @@ -643,19 +641,19 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons taosMemoryFree(buffer); } -static void backUpSlowLog(){ +static void checkSwitchSlowLogFile(){ char buf[64] = {0}; time_t t = taosTime(NULL); struct tm tmInfo; if (taosLocalTime(&t, &tmInfo, buf) != NULL) { (void)strftime(buf, sizeof(buf), "%Y-%m-%d", &tmInfo); } -// (void)taosThreadMutexLock(&tsLogObj.logMutex); -// if (strlen(tsLogObj.slowLogDay) == 0) { -// tstrncpy(tsLogObj.slowLogDay, buf, 64); -// }else if (strcmp(tsLogObj.slowLogDay, buf) != 0) { - (void)taosOpenNewSlowLogFile(buf); -// } + (void)taosThreadMutexLock(&tsLogObj.logMutex); + if (strlen(tsLogObj.slowLogDay) == 0) { + tstrncpy(tsLogObj.slowLogDay, buf, 64); + }else if (strcmp(tsLogObj.slowLogDay, buf) != 0) { + taosOpenNewSlowLogFile(buf); + } (void)taosThreadMutexUnlock(&tsLogObj.logMutex); } void taosPrintSlowLog(const char *format, ...) { @@ -683,6 +681,8 @@ void taosPrintSlowLog(const char *format, ...) { (void)taosWriteFile(tsLogObj.slowHandle->pFile, buffer, len); } + checkSwitchSlowLogFile(); + taosMemoryFree(buffer); } From 0b663dbcccc09722d8f626367a34528b11d4c941 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 2 Aug 2024 17:11:32 +0800 Subject: [PATCH 3/8] feat:[TS-3718]save slow log file one day one file --- source/util/src/tlog.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 199e030d82..34893817b5 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -135,7 +135,6 @@ static int32_t taosPushLogBuffer(SLogBuff *pLogBuf, const char *msg, int32_t m static SLogBuff *taosLogBuffNew(int32_t bufSize); static void taosCloseLogByFd(TdFilePtr pFile); static int32_t taosInitNormalLog(const char *fn, int32_t maxFileNum); -static void backUpSlowLog(); static int32_t taosStartLog() { TdThreadAttr threadAttr; (void)taosThreadAttrInit(&threadAttr); From 51ad50c60362f5adf28bd3c8f428657019133bf2 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 2 Aug 2024 17:16:13 +0800 Subject: [PATCH 4/8] feat:[TS-3718]save slow log file one day one file --- source/util/src/tlog.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 34893817b5..bb924a5c90 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -573,7 +573,7 @@ static inline int32_t taosBuildLogHead(char *buffer, const char *flags) { } static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *buffer, int32_t len) { - if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile != NULL && osLogSpaceAvailable()) { + if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile != NULL && osLogSpaceSufficient()) { taosUpdateLogNums(level); if (tsAsyncLog) { (void)taosPushLogBuffer(tsLogObj.logHandle, buffer, len); @@ -621,7 +621,7 @@ void taosPrintLog(const char *flags, ELogLevel level, int32_t dflag, const char } void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, const char *format, ...) { - if (!osLogSpaceAvailable()) return; + if (!osLogSpaceSufficient()) return; if (!(dflag & DEBUG_FILE) && !(dflag & DEBUG_SCREEN)) return; char *buffer = taosMemoryMalloc(LOG_MAX_LINE_DUMP_BUFFER_SIZE); @@ -656,7 +656,7 @@ static void checkSwitchSlowLogFile(){ (void)taosThreadMutexUnlock(&tsLogObj.logMutex); } void taosPrintSlowLog(const char *format, ...) { - if (!osLogSpaceAvailable()) return; + if (!osLogSpaceSufficient()) return; char *buffer = taosMemoryMalloc(LOG_MAX_LINE_DUMP_BUFFER_SIZE); int32_t len = taosBuildLogHead(buffer, ""); From 4d7c54d1a5233960df45304488ed10dd99bf5f3b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 2 Aug 2024 19:10:50 +0800 Subject: [PATCH 5/8] feat:[TS-3718]save slow log file one day one file --- source/util/src/tlog.c | 117 ++++++++++++++++++----------------------- 1 file changed, 50 insertions(+), 67 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index bb924a5c90..5eacf561aa 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -26,7 +26,8 @@ #define LOG_MAX_LINE_DUMP_SIZE (1024 * 1024) #define LOG_MAX_LINE_DUMP_BUFFER_SIZE (LOG_MAX_LINE_DUMP_SIZE + 128) -#define LOG_FILE_NAME_LEN 300 +#define LOG_FILE_DAY_LEN 64 + #define LOG_DEFAULT_BUF_SIZE (20 * 1024 * 1024) // 20MB #define LOG_SLOW_BUF_SIZE (10 * 1024 * 1024) // 10MB @@ -69,10 +70,10 @@ typedef struct { int32_t openInProgress; int32_t openInProgressSlowLog; int64_t lastKeepFileSec; - char slowLogDay[64]; + char slowLogDay[LOG_FILE_DAY_LEN]; pid_t pid; - char logName[LOG_FILE_NAME_LEN]; - char slowLogName[LOG_FILE_NAME_LEN]; + char logName[PATH_MAX]; + char slowLogName[PATH_MAX]; SLogBuff *logHandle; SLogBuff *slowHandle; TdThreadMutex logMutex; @@ -145,6 +146,13 @@ static int32_t taosStartLog() { return 0; } +static void getDay(char* buf){ + time_t t = taosTime(NULL); + struct tm tmInfo; + if (taosLocalTime(&t, &tmInfo, buf) != NULL) { + (void)strftime(buf, sizeof(buf), "%Y-%m-%d", &tmInfo); + } +} static void getFullPathName(char* fullName, const char* logName){ if (strlen(tsLogDir) != 0) { char lastC = tsLogDir[strlen(tsLogDir) - 1]; @@ -164,20 +172,25 @@ static void getFullPathName(char* fullName, const char* logName){ int32_t taosInitSlowLog() { char logFileName[64] = {0}; #ifdef CUS_PROMPT - snprintf(logFileName, 64, "%sSlowLog", CUS_PROMPT); + (void)snprintf(logFileName, 64, "%sSlowLog", CUS_PROMPT); #else - snprintf(logFileName, 64, "taosSlowLog"); + (void)snprintf(logFileName, 64, "taosSlowLog"); #endif getFullPathName(tsLogObj.slowLogName, logFileName); + char name[PATH_MAX + LOG_FILE_DAY_LEN] = {0}; + char day[LOG_FILE_DAY_LEN] = {0}; + getDay(day); + (void)snprintf(name, PATH_MAX + LOG_FILE_DAY_LEN, "%s.%s", tsLogObj.slowLogName, day); + tsLogObj.slowHandle = taosLogBuffNew(LOG_SLOW_BUF_SIZE); if (tsLogObj.slowHandle == NULL) return terrno; (void)taosUmaskFile(0); - tsLogObj.slowHandle->pFile = taosOpenFile(tsLogObj.slowLogName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + tsLogObj.slowHandle->pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); if (tsLogObj.slowHandle->pFile == NULL) { - printf("\nfailed to open slow log file:%s, reason:%s\n", tsLogObj.slowLogName, strerror(errno)); + printf("\nfailed to open slow log file:%s, reason:%s\n", name, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } @@ -266,7 +279,7 @@ static void taosReserveOldLog(char *oldName, char *keepName) { } else { fileSec = ++tsLogObj.lastKeepFileSec; } - snprintf(keepName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64, tsLogObj.logName, fileSec); + snprintf(keepName, PATH_MAX + 20, "%s.%" PRId64, tsLogObj.logName, fileSec); if ((code = taosRenameFile(oldName, keepName))) { keepName[0] = 0; uError("failed to rename file:%s to %s since %s", oldName, keepName, tstrerror(code)); @@ -275,8 +288,8 @@ static void taosReserveOldLog(char *oldName, char *keepName) { static void taosKeepOldLog(char *oldName) { if (oldName[0] != 0) { - char compressFileName[LOG_FILE_NAME_LEN + 20]; - snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.gz", oldName); + char compressFileName[PATH_MAX + 20]; + snprintf(compressFileName, PATH_MAX + 20, "%s.gz", oldName); if (taosCompressFile(oldName, compressFileName) == 0) { (void)taosRemoveFile(oldName); } @@ -288,15 +301,15 @@ static void taosKeepOldLog(char *oldName) { } typedef struct { TdFilePtr pOldFile; - char keepName[LOG_FILE_NAME_LEN + 20]; + char keepName[PATH_MAX + 20]; } OldFileKeeper; static OldFileKeeper *taosOpenNewFile() { - char keepName[LOG_FILE_NAME_LEN + 20]; + char keepName[PATH_MAX + 20]; sprintf(keepName, "%s.%d", tsLogObj.logName, tsLogObj.flag); tsLogObj.flag ^= 1; tsLogObj.lines = 0; - char name[LOG_FILE_NAME_LEN + 20]; + char name[PATH_MAX + 20]; sprintf(name, "%s.%d", tsLogObj.logName, tsLogObj.flag); (void)taosUmaskFile(0); @@ -362,45 +375,19 @@ static int32_t taosOpenNewLogFile() { } static void taosOpenNewSlowLogFile(char* day) { - int32_t code = 0; - TdFilePtr pOldFile = tsLogObj.slowHandle->pFile; - if (taosLockFile(pOldFile) != 0){ + TdFilePtr pFile = NULL; + char name[PATH_MAX + LOG_FILE_DAY_LEN] = {0}; + (void)snprintf(name, PATH_MAX + LOG_FILE_DAY_LEN, "%s.%s", tsLogObj.slowLogName, day); + pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pFile == NULL) { + uError("open new log file fail! reason:%s, reuse lastlog", strerror(errno)); return; } - TdFilePtr pFile = NULL; - char name[LOG_FILE_NAME_LEN + 64] = {0}; - (void)sprintf(name, "%s.%s", tsLogObj.slowLogName, day); - if (taosCheckExistFile(name)){ - pFile = taosOpenFile(tsLogObj.slowLogName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); - if (pFile == NULL) { - uError("open new log file fail! reason:%s, reuse lastlog", strerror(errno)); - goto END; - } - }else{ - if ((code = taosRenameFile(tsLogObj.slowLogName, name))) { - uError("failed to rename file:%s to %s since %s", tsLogObj.slowLogName, name, tstrerror(code)); - goto END; - } - - (void)taosUmaskFile(0); - - pFile = taosOpenFile(tsLogObj.slowLogName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - uError("open new log file fail! reason:%s, reuse lastlog", strerror(errno)); - goto END; - } - (void)taosLSeekFile(pFile, 0, SEEK_SET); - } - + TdFilePtr pOldFile = tsLogObj.slowHandle->pFile; tsLogObj.slowHandle->pFile = pFile; - taosUnLockLogFile(pOldFile); (void)taosCloseFile(&pOldFile); - tstrncpy(tsLogObj.slowLogDay, day, 64); - return; - -END: - taosUnLockLogFile(pOldFile); + tstrncpy(tsLogObj.slowLogDay, day, LOG_FILE_DAY_LEN); } void taosResetLog() { @@ -439,34 +426,34 @@ static void decideLogFileName(const char *fn, int32_t maxFileNum) { tsLogObj.fileNum = maxFileNum; if (tsLogObj.fileNum > 1) { for (int32_t i = 0; i < tsLogObj.fileNum; i++) { - char fileName[LOG_FILE_NAME_LEN]; + char fileName[PATH_MAX + 10]; - snprintf(fileName, LOG_FILE_NAME_LEN, "%s%d.0", fn, i); + (void)snprintf(fileName, PATH_MAX + 10, "%s%d.0", fn, i); bool file1open = taosCheckFileIsOpen(fileName); - snprintf(fileName, LOG_FILE_NAME_LEN, "%s%d.1", fn, i); + (void)snprintf(fileName, PATH_MAX + 10, "%s%d.1", fn, i); bool file2open = taosCheckFileIsOpen(fileName); if (!file1open && !file2open) { - snprintf(tsLogObj.logName, LOG_FILE_NAME_LEN, "%s%d", fn, i); + (void)snprintf(tsLogObj.logName, PATH_MAX, "%s%d", fn, i); return; } } } - if (strlen(fn) < LOG_FILE_NAME_LEN) { + if (strlen(fn) < PATH_MAX) { strcpy(tsLogObj.logName, fn); } } static void decideLogFileNameFlag(){ - char name[LOG_FILE_NAME_LEN + 50] = "\0"; + char name[PATH_MAX + 50] = "\0"; int32_t logstat0_mtime = 0; int32_t logstat1_mtime = 0; bool log0Exist = false; bool log1Exist = false; - if (strlen(tsLogObj.logName) < LOG_FILE_NAME_LEN + 50 - 2) { + if (strlen(tsLogObj.logName) < PATH_MAX + 50 - 2) { strcpy(name, tsLogObj.logName); strcat(name, ".0"); log0Exist = taosStatFile(name, NULL, &logstat0_mtime, NULL) >= 0; @@ -504,7 +491,7 @@ static int32_t taosInitNormalLog(const char *logName, int32_t maxFileNum) { processLogFileName(logName, maxFileNum); - char name[LOG_FILE_NAME_LEN + 50] = "\0"; + char name[PATH_MAX + 50] = "\0"; sprintf(name, "%s.%d", tsLogObj.logName, tsLogObj.flag); (void)taosThreadMutexInit(&tsLogObj.logMutex, NULL); @@ -641,17 +628,13 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons } static void checkSwitchSlowLogFile(){ - char buf[64] = {0}; - time_t t = taosTime(NULL); - struct tm tmInfo; - if (taosLocalTime(&t, &tmInfo, buf) != NULL) { - (void)strftime(buf, sizeof(buf), "%Y-%m-%d", &tmInfo); - } + char day[LOG_FILE_DAY_LEN] = {0}; + getDay(day); (void)taosThreadMutexLock(&tsLogObj.logMutex); if (strlen(tsLogObj.slowLogDay) == 0) { - tstrncpy(tsLogObj.slowLogDay, buf, 64); - }else if (strcmp(tsLogObj.slowLogDay, buf) != 0) { - taosOpenNewSlowLogFile(buf); + tstrncpy(tsLogObj.slowLogDay, day, LOG_FILE_DAY_LEN); + }else if (strcmp(tsLogObj.slowLogDay, day) != 0) { + taosOpenNewSlowLogFile(day); } (void)taosThreadMutexUnlock(&tsLogObj.logMutex); } @@ -674,14 +657,14 @@ void taosPrintSlowLog(const char *format, ...) { (void)atomic_add_fetch_64(&tsNumOfSlowLogs, 1); + checkSwitchSlowLogFile(); + if (tsAsyncLog) { (void)taosPushLogBuffer(tsLogObj.slowHandle, buffer, len); } else { (void)taosWriteFile(tsLogObj.slowHandle->pFile, buffer, len); } - checkSwitchSlowLogFile(); - taosMemoryFree(buffer); } From 9c2adc59bb1ab2638b46915e127717233dd4a56e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 5 Aug 2024 15:52:22 +0800 Subject: [PATCH 6/8] feat:[TS-3718]save slow log file one day one file --- source/util/src/tlog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 5eacf561aa..c6c869fb23 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -150,7 +150,7 @@ static void getDay(char* buf){ time_t t = taosTime(NULL); struct tm tmInfo; if (taosLocalTime(&t, &tmInfo, buf) != NULL) { - (void)strftime(buf, sizeof(buf), "%Y-%m-%d", &tmInfo); + (void)strftime(buf, LOG_FILE_DAY_LEN, "%Y-%m-%d", &tmInfo); } } static void getFullPathName(char* fullName, const char* logName){ From 7f8df84106cff9a175e95a86e01211acb7aafbfa Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 6 Aug 2024 18:02:01 +0800 Subject: [PATCH 7/8] feat:[TS-3718]save slow log file one day one file --- source/util/src/tlog.c | 54 +++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index c6c869fb23..26e5a0039d 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -68,9 +68,8 @@ typedef struct { int32_t lines; int32_t flag; int32_t openInProgress; - int32_t openInProgressSlowLog; int64_t lastKeepFileSec; - char slowLogDay[LOG_FILE_DAY_LEN]; + int64_t timestampToday; pid_t pid; char logName[PATH_MAX]; char slowLogName[PATH_MAX]; @@ -136,6 +135,7 @@ static int32_t taosPushLogBuffer(SLogBuff *pLogBuf, const char *msg, int32_t m static SLogBuff *taosLogBuffNew(int32_t bufSize); static void taosCloseLogByFd(TdFilePtr pFile); static int32_t taosInitNormalLog(const char *fn, int32_t maxFileNum); +static void taosWriteLog(SLogBuff *pLogBuf); static int32_t taosStartLog() { TdThreadAttr threadAttr; (void)taosThreadAttrInit(&threadAttr); @@ -153,6 +153,18 @@ static void getDay(char* buf){ (void)strftime(buf, LOG_FILE_DAY_LEN, "%Y-%m-%d", &tmInfo); } } + +static int64_t getTimestampToday() { + time_t t = taosTime(NULL); + struct tm tm; + (void) taosLocalTime(&t, &tm, NULL); + tm.tm_hour = 0; + tm.tm_min = 0; + tm.tm_sec = 0; + + return (int64_t)taosMktime(&tm); +} + static void getFullPathName(char* fullName, const char* logName){ if (strlen(tsLogDir) != 0) { char lastC = tsLogDir[strlen(tsLogDir) - 1]; @@ -184,6 +196,7 @@ int32_t taosInitSlowLog() { getDay(day); (void)snprintf(name, PATH_MAX + LOG_FILE_DAY_LEN, "%s.%s", tsLogObj.slowLogName, day); + tsLogObj.timestampToday = getTimestampToday(); tsLogObj.slowHandle = taosLogBuffNew(LOG_SLOW_BUF_SIZE); if (tsLogObj.slowHandle == NULL) return terrno; @@ -374,20 +387,31 @@ static int32_t taosOpenNewLogFile() { return 0; } -static void taosOpenNewSlowLogFile(char* day) { +static void taosOpenNewSlowLogFile(int64_t today) { + (void)taosThreadMutexLock(&tsLogObj.logMutex); + if (tsLogObj.timestampToday == today) { + uInfo("timestampToday is already equal to today, no need to open new slow log file"); + (void)taosThreadMutexUnlock(&tsLogObj.logMutex); + return; + } + taosWriteLog(tsLogObj.slowHandle); + char day[LOG_FILE_DAY_LEN] = {0}; + getDay(day); TdFilePtr pFile = NULL; char name[PATH_MAX + LOG_FILE_DAY_LEN] = {0}; (void)snprintf(name, PATH_MAX + LOG_FILE_DAY_LEN, "%s.%s", tsLogObj.slowLogName, day); pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); if (pFile == NULL) { uError("open new log file fail! reason:%s, reuse lastlog", strerror(errno)); + (void)taosThreadMutexUnlock(&tsLogObj.logMutex); return; } TdFilePtr pOldFile = tsLogObj.slowHandle->pFile; tsLogObj.slowHandle->pFile = pFile; (void)taosCloseFile(&pOldFile); - tstrncpy(tsLogObj.slowLogDay, day, LOG_FILE_DAY_LEN); + tsLogObj.timestampToday = today; + (void)taosThreadMutexUnlock(&tsLogObj.logMutex); } void taosResetLog() { @@ -456,9 +480,9 @@ static void decideLogFileNameFlag(){ if (strlen(tsLogObj.logName) < PATH_MAX + 50 - 2) { strcpy(name, tsLogObj.logName); strcat(name, ".0"); - log0Exist = taosStatFile(name, NULL, &logstat0_mtime, NULL) >= 0; + log0Exist = taosStatFile(name, NULL, &logstat0_mtime, NULL) == 0; name[strlen(name) - 1] = '1'; - log1Exist = taosStatFile(name, NULL, &logstat1_mtime, NULL) >= 0; + log1Exist = taosStatFile(name, NULL, &logstat1_mtime, NULL) == 0; } // if none of the log files exist, open 0, if both exists, open the old one @@ -627,20 +651,14 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons taosMemoryFree(buffer); } -static void checkSwitchSlowLogFile(){ - char day[LOG_FILE_DAY_LEN] = {0}; - getDay(day); - (void)taosThreadMutexLock(&tsLogObj.logMutex); - if (strlen(tsLogObj.slowLogDay) == 0) { - tstrncpy(tsLogObj.slowLogDay, day, LOG_FILE_DAY_LEN); - }else if (strcmp(tsLogObj.slowLogDay, day) != 0) { - taosOpenNewSlowLogFile(day); - } - (void)taosThreadMutexUnlock(&tsLogObj.logMutex); -} void taosPrintSlowLog(const char *format, ...) { if (!osLogSpaceSufficient()) return; + int64_t today = getTimestampToday(); + if (today != tsLogObj.timestampToday){ + taosOpenNewSlowLogFile(today); + } + char *buffer = taosMemoryMalloc(LOG_MAX_LINE_DUMP_BUFFER_SIZE); int32_t len = taosBuildLogHead(buffer, ""); @@ -657,8 +675,6 @@ void taosPrintSlowLog(const char *format, ...) { (void)atomic_add_fetch_64(&tsNumOfSlowLogs, 1); - checkSwitchSlowLogFile(); - if (tsAsyncLog) { (void)taosPushLogBuffer(tsLogObj.slowHandle, buffer, len); } else { From bce6f40abbc7390d34048556411e114d66a65af2 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 8 Aug 2024 17:29:18 +0800 Subject: [PATCH 8/8] feat:[TS-3718]add lock to avoid write slow at the same time --- source/util/src/tlog.c | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 26e5a0039d..411e251706 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -61,6 +61,7 @@ typedef struct { TdThreadMutex buffMutex; int32_t writeInterval; int32_t lastDuration; + int32_t lock; } SLogBuff; typedef struct { @@ -136,6 +137,8 @@ static SLogBuff *taosLogBuffNew(int32_t bufSize); static void taosCloseLogByFd(TdFilePtr pFile); static int32_t taosInitNormalLog(const char *fn, int32_t maxFileNum); static void taosWriteLog(SLogBuff *pLogBuf); +static void taosWriteSlowLog(SLogBuff *pLogBuf); + static int32_t taosStartLog() { TdThreadAttr threadAttr; (void)taosThreadAttrInit(&threadAttr); @@ -387,14 +390,24 @@ static int32_t taosOpenNewLogFile() { return 0; } -static void taosOpenNewSlowLogFile(int64_t today) { +static void taosOpenNewSlowLogFile() { (void)taosThreadMutexLock(&tsLogObj.logMutex); - if (tsLogObj.timestampToday == today) { + int64_t delta = taosGetTimestampSec() - tsLogObj.timestampToday; + if (delta >= 0 && delta < 86400) { uInfo("timestampToday is already equal to today, no need to open new slow log file"); (void)taosThreadMutexUnlock(&tsLogObj.logMutex); return; } + + for (int32_t i = 1; atomic_val_compare_exchange_32(&tsLogObj.slowHandle->lock, 0, 1) == 1; ++i) { + if (i % 1000 == 0) { + (void)sched_yield(); + } + } + tsLogObj.slowHandle->lastDuration = LOG_MAX_WAIT_MSEC; // force write taosWriteLog(tsLogObj.slowHandle); + atomic_store_32(&tsLogObj.slowHandle->lock, 0); + char day[LOG_FILE_DAY_LEN] = {0}; getDay(day); TdFilePtr pFile = NULL; @@ -410,7 +423,7 @@ static void taosOpenNewSlowLogFile(int64_t today) { TdFilePtr pOldFile = tsLogObj.slowHandle->pFile; tsLogObj.slowHandle->pFile = pFile; (void)taosCloseFile(&pOldFile); - tsLogObj.timestampToday = today; + tsLogObj.timestampToday = getTimestampToday(); (void)taosThreadMutexUnlock(&tsLogObj.logMutex); } @@ -654,9 +667,9 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons void taosPrintSlowLog(const char *format, ...) { if (!osLogSpaceSufficient()) return; - int64_t today = getTimestampToday(); - if (today != tsLogObj.timestampToday){ - taosOpenNewSlowLogFile(today); + int64_t delta = taosGetTimestampSec() - tsLogObj.timestampToday; + if (delta >= 86400 || delta < 0) { + taosOpenNewSlowLogFile(); } char *buffer = taosMemoryMalloc(LOG_MAX_LINE_DUMP_BUFFER_SIZE); @@ -729,7 +742,7 @@ static SLogBuff *taosLogBuffNew(int32_t bufSize) { pLogBuf->minBuffSize = bufSize / 10; pLogBuf->stop = 0; pLogBuf->writeInterval = LOG_DEFAULT_INTERVAL; - + pLogBuf->lock = 0; if (taosThreadMutexInit(&LOG_BUF_MUTEX(pLogBuf), NULL) < 0) goto _err; // tsem_init(&(pLogBuf->buffNotEmpty), 0, 0); @@ -811,6 +824,12 @@ static int32_t taosGetLogRemainSize(SLogBuff *pLogBuf, int32_t start, int32_t en return rSize >= 0 ? rSize : LOG_BUF_SIZE(pLogBuf) + rSize; } +static void taosWriteSlowLog(SLogBuff *pLogBuf){ + int32_t lock = atomic_val_compare_exchange_32(&pLogBuf->lock, 0, 1); + if (lock == 1) return; + taosWriteLog(pLogBuf); + atomic_store_32(&pLogBuf->lock, 0); +} static void taosWriteLog(SLogBuff *pLogBuf) { int32_t start = LOG_BUF_START(pLogBuf); int32_t end = LOG_BUF_END(pLogBuf); @@ -895,12 +914,12 @@ static void *taosAsyncOutputLog(void *param) { // Polling the buffer taosWriteLog(pLogBuf); - if (pSlowBuf) taosWriteLog(pSlowBuf); + if (pSlowBuf) taosWriteSlowLog(pSlowBuf); if (pLogBuf->stop || (pSlowBuf && pSlowBuf->stop)) { pLogBuf->lastDuration = LOG_MAX_WAIT_MSEC; taosWriteLog(pLogBuf); - if (pSlowBuf) taosWriteLog(pSlowBuf); + if (pSlowBuf) taosWriteSlowLog(pSlowBuf); break; } }