feat:[TS-3718]save slow log file one day one file

This commit is contained in:
wangmm0220 2024-08-02 16:29:48 +08:00
parent c75281b9ff
commit 4d19f48d8a
25 changed files with 167 additions and 120 deletions

View File

@ -69,7 +69,7 @@ extern int32_t tdbDebugFlag;
extern int32_t sndDebugFlag;
extern int32_t simDebugFlag;
int32_t taosInitLog(const char *logName, int32_t maxFiles);
int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc);
void taosCloseLog();
void taosResetLog();
void taosDumpData(uint8_t *msg, int32_t len);

View File

@ -1596,7 +1596,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
goto _exit;
}
if ((code = taosInitLog(logname, logFileNum)) != 0) {
if ((code = taosInitLog(logname, logFileNum, tsc)) != 0) {
printf("failed to init log file since %s\n", tstrerror(code));
goto _exit;
}

View File

@ -37,7 +37,7 @@ void Testbase::InitLog(const char* path) {
taosGetSystemInfo();
tsRpcQueueMemoryAllowed = tsTotalMemoryKB * 0.1;
if (taosInitLog("taosdlog", 1) != 0) {
if (taosInitLog("taosdlog", 1, false) != 0) {
printf("failed to init log file\n");
}
}

View File

@ -35,7 +35,7 @@ class MndTestSdb : public ::testing::Test {
taosRemoveDir(path);
taosMkDir(path);
tstrncpy(tsLogDir, path, PATH_MAX);
if (taosInitLog("taosdlog", 1) != 0) {
if (taosInitLog("taosdlog", 1, false) != 0) {
printf("failed to init log file\n");
}
}

View File

@ -52,7 +52,7 @@ class MndTestTrans2 : public ::testing::Test {
taosRemoveDir(logpath);
taosMkDir(logpath);
tstrncpy(tsLogDir, logpath, PATH_MAX);
if (taosInitLog("taosdlog", 1) != 0) {
if (taosInitLog("taosdlog", 1, false) != 0) {
printf("failed to init log file\n");
}
}

View File

@ -2884,7 +2884,7 @@ void jtInitLogFile() {
qDebugFlag = 159;
TAOS_STRCPY(tsLogDir, TD_LOG_DIR_PATH);
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) {
JT_PRINTF("failed to open log file in directory:%s\n", tsLogDir);
}
}

View File

@ -28,7 +28,7 @@ static void EnvInit() {
taosMkDir(path.c_str());
// init log file
tstrncpy(tsLogDir, path.c_str(), PATH_MAX);
if (taosInitLog("tindex.idx", 1) != 0) {
if (taosInitLog("tindex.idx", 1, false) != 0) {
printf("failed to init log");
}
// init index file

View File

@ -40,7 +40,7 @@ static void initLog() {
taosRemoveDir(tsLogDir);
taosMkDir(tsLogDir);
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir);
}
}

View File

@ -283,7 +283,7 @@ static void initLog() {
taosRemoveDir(tsLogDir);
taosMkDir(tsLogDir);
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir);
}
}

View File

@ -57,7 +57,7 @@ void sifInitLogFile() {
taosRemoveDir(tsLogDir);
taosMkDir(tsLogDir);
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir);
}
}

View File

@ -40,7 +40,7 @@ static void initLog() {
taosRemoveDir(tsLogDir);
taosMkDir(tsLogDir);
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir);
}
}

View File

@ -72,7 +72,7 @@ class ParserEnv : public testing::Environment {
taosRemoveDir(path);
ASSERT_EQ(TSDB_CODE_SUCCESS, taosMkDir(path));
tstrncpy(tsLogDir, path, PATH_MAX);
if (taosInitLog("taoslog", 1) != 0) {
if (taosInitLog("taoslog", 1, false) != 0) {
std::cout << "failed to init log file" << std::endl;
}
}

View File

@ -67,7 +67,7 @@ class PlannerEnv : public testing::Environment {
taosRemoveDir(path);
ASSERT_EQ(TSDB_CODE_SUCCESS,taosMkDir(path));
tstrncpy(tsLogDir, path, PATH_MAX);
if (taosInitLog("taoslog", 1) != 0) {
if (taosInitLog("taoslog", 1, false) != 0) {
std::cout << "failed to init log file" << std::endl;
}
}

View File

@ -105,7 +105,7 @@ void qwtInitLogFile() {
qDebugFlag = 159;
strcpy(tsLogDir, TD_LOG_DIR_PATH);
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir);
}
}

View File

@ -57,7 +57,7 @@ void flttInitLogFile() {
qDebugFlag = 159;
(void)strcpy(tsLogDir, TD_LOG_DIR_PATH);
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir);
}
}

View File

@ -83,7 +83,7 @@ void scltInitLogFile() {
qDebugFlag = 159;
(void)strcpy(tsLogDir, TD_LOG_DIR_PATH);
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) {
(void)printf("failed to open log file in directory:%s\n", tsLogDir);
}
}

View File

@ -79,7 +79,7 @@ void schtInitLogFile() {
qDebugFlag = 159;
TAOS_STRCPY(tsLogDir, TD_LOG_DIR_PATH);
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) {
(void)printf("failed to open log file in directory:%s\n", tsLogDir);
}
}

View File

@ -48,7 +48,7 @@ void test4() {
}
int main(int argc, char** argv) {
taosInitLog("/tmp/syncTest.log", 100);
taosInitLog("/tmp/syncTest.log", 100, false);
tsAsyncLog = 0;
sDebugFlag = DEBUG_SCREEN + DEBUG_FILE + DEBUG_TRACE + DEBUG_INFO + DEBUG_ERROR;

View File

@ -351,7 +351,7 @@ int main(int argc, char** argv) {
char logFile[256];
snprintf(logFile, sizeof(logFile), "/tmp/%s-replicaNum%d-myIndex%d.log", gDir, replicaNum, myIndex);
taosInitLog(logFile, 100);
taosInitLog(logFile, 100, false);
sTrace("logFile : %s", logFile);
gSnapshotLastApplyIndex = lastApplyIndex;

View File

@ -43,7 +43,7 @@ void initLogEnv() {
taosRemoveDir(tsLogDir);
taosMkDir(tsLogDir);
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir);
}
}

View File

@ -31,7 +31,7 @@ void initLogEnv() {
taosRemoveDir(tsLogDir);
taosMkDir(tsLogDir);
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir);
}
}

View File

@ -53,7 +53,7 @@ void initLogEnv() {
taosRemoveDir(tsLogDir);
taosMkDir(tsLogDir);
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir);
}
}

View File

@ -230,7 +230,7 @@ static void initEnv() {
taosMkDir(path.c_str());
tstrncpy(tsLogDir, path.c_str(), PATH_MAX);
if (taosInitLog("taosdlog", 1) != 0) {
if (taosInitLog("taosdlog", 1, false) != 0) {
printf("failed to init log file\n");
}
}

View File

@ -67,9 +67,12 @@ typedef struct {
int32_t lines;
int32_t flag;
int32_t openInProgress;
int32_t openInProgressSlowLog;
int64_t lastKeepFileSec;
char slowLogDay[64];
pid_t pid;
char logName[LOG_FILE_NAME_LEN];
char slowLogName[LOG_FILE_NAME_LEN];
SLogBuff *logHandle;
SLogBuff *slowHandle;
TdThreadMutex logMutex;
@ -77,7 +80,7 @@ typedef struct {
extern SConfig *tsCfg;
static int8_t tsLogInited = 0;
static SLogObj tsLogObj = {.fileNum = 1};
static SLogObj tsLogObj = {.fileNum = 1, .slowHandle = NULL};
static int64_t tsAsyncLogLostLines = 0;
static int32_t tsDaylightActive; /* Currently in daylight saving time. */
@ -131,18 +134,8 @@ static void *taosAsyncOutputLog(void *param);
static int32_t taosPushLogBuffer(SLogBuff *pLogBuf, const char *msg, int32_t msgLen);
static SLogBuff *taosLogBuffNew(int32_t bufSize);
static void taosCloseLogByFd(TdFilePtr pFile);
static int32_t taosOpenLogFile(char *fn, int32_t maxFileNum);
static FORCE_INLINE void taosUpdateDaylight() {
struct tm Tm, *ptm;
struct timeval timeSecs;
(void)taosGetTimeOfDay(&timeSecs);
time_t curTime = timeSecs.tv_sec;
ptm = taosLocalTime(&curTime, &Tm, NULL);
tsDaylightActive = ptm->tm_isdst;
}
static FORCE_INLINE int32_t taosGetDaylight() { return tsDaylightActive; }
static int32_t taosInitNormalLog(const char *fn, int32_t maxFileNum);
static void backUpSlowLog();
static int32_t taosStartLog() {
TdThreadAttr threadAttr;
(void)taosThreadAttrInit(&threadAttr);
@ -153,47 +146,7 @@ static int32_t taosStartLog() {
return 0;
}
int32_t taosInitSlowLog() {
char fullName[PATH_MAX] = {0};
char logFileName[64] = {0};
#ifdef CUS_PROMPT
snprintf(logFileName, 64, "%sSlowLog", CUS_PROMPT);
#else
snprintf(logFileName, 64, "taosSlowLog");
#endif
if (strlen(tsLogDir) != 0) {
char lastC = tsLogDir[strlen(tsLogDir) - 1];
if (lastC == '\\' || lastC == '/') {
snprintf(fullName, PATH_MAX,
"%s"
"%s",
tsLogDir, logFileName);
} else {
snprintf(fullName, PATH_MAX, "%s" TD_DIRSEP "%s", tsLogDir, logFileName);
}
} else {
snprintf(fullName, PATH_MAX, "%s", logFileName);
}
tsLogObj.slowHandle = taosLogBuffNew(LOG_SLOW_BUF_SIZE);
if (tsLogObj.slowHandle == NULL) return terrno;
(void)taosUmaskFile(0);
tsLogObj.slowHandle->pFile = taosOpenFile(fullName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (tsLogObj.slowHandle->pFile == NULL) {
printf("\nfailed to open slow log file:%s, reason:%s\n", fullName, strerror(errno));
return TAOS_SYSTEM_ERROR(errno);
}
return 0;
}
int32_t taosInitLog(const char *logName, int32_t maxFiles) {
if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0;
osUpdate();
char fullName[PATH_MAX] = {0};
static void getFullPathName(char* fullName, const char* logName){
if (strlen(tsLogDir) != 0) {
char lastC = tsLogDir[strlen(tsLogDir) - 1];
if (lastC == '\\' || lastC == '/') {
@ -207,13 +160,40 @@ int32_t taosInitLog(const char *logName, int32_t maxFiles) {
} else {
snprintf(fullName, PATH_MAX, "%s", logName);
}
taosUpdateDaylight();
}
tsLogObj.logHandle = taosLogBuffNew(LOG_DEFAULT_BUF_SIZE);
if (tsLogObj.logHandle == NULL) return terrno;
TAOS_CHECK_RETURN(taosOpenLogFile(fullName, maxFiles));
int32_t taosInitSlowLog() {
char logFileName[64] = {0};
#ifdef CUS_PROMPT
snprintf(logFileName, 64, "%sSlowLog", CUS_PROMPT);
#else
snprintf(logFileName, 64, "taosSlowLog");
#endif
TAOS_CHECK_RETURN(taosInitSlowLog());
getFullPathName(tsLogObj.slowLogName, logFileName);
tsLogObj.slowHandle = taosLogBuffNew(LOG_SLOW_BUF_SIZE);
if (tsLogObj.slowHandle == NULL) return terrno;
(void)taosUmaskFile(0);
tsLogObj.slowHandle->pFile = taosOpenFile(tsLogObj.slowLogName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (tsLogObj.slowHandle->pFile == NULL) {
printf("\nfailed to open slow log file:%s, reason:%s\n", tsLogObj.slowLogName, strerror(errno));
return TAOS_SYSTEM_ERROR(errno);
}
backUpSlowLog();
return 0;
}
int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) {
if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0;
osUpdate();
TAOS_CHECK_RETURN(taosInitNormalLog(logName, maxFiles));
if (tsc){
TAOS_CHECK_RETURN(taosInitSlowLog());
}
TAOS_CHECK_RETURN(taosStartLog());
return 0;
}
@ -383,6 +363,49 @@ static int32_t taosOpenNewLogFile() {
return 0;
}
static void taosOpenNewSlowLogFile(char* day) {
int32_t code = 0;
TdFilePtr pOldFile = tsLogObj.slowHandle->pFile;
if (taosLockFile(pOldFile) != 0){
return;
}
TdFilePtr pFile = NULL;
char name[LOG_FILE_NAME_LEN + 64] = {0};
(void)sprintf(name, "%s.%s", tsLogObj.slowLogName, day);
if (taosCheckExistFile(name)){
pFile = taosOpenFile(tsLogObj.slowLogName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pFile == NULL) {
uError("open new log file fail! reason:%s, reuse lastlog", strerror(errno));
goto END;
}
}else{
if ((code = taosRenameFile(tsLogObj.slowLogName, name))) {
uError("failed to rename file:%s to %s since %s", tsLogObj.slowLogName, name, tstrerror(code));
goto END;
}
(void)taosUmaskFile(0);
pFile = taosOpenFile(tsLogObj.slowLogName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
uError("open new log file fail! reason:%s, reuse lastlog", strerror(errno));
goto END;
}
(void)taosLSeekFile(pFile, 0, SEEK_SET);
}
tsLogObj.slowHandle->pFile = pFile;
taosUnLockLogFile(pOldFile);
(void)taosCloseFile(&pOldFile);
tstrncpy(tsLogObj.slowLogDay, day, 64);
return;
END:
taosUnLockLogFile(pOldFile);
}
void taosResetLog() {
// force create a new log file
tsLogObj.lines = tsNumOfLogLines + 10;
@ -415,7 +438,8 @@ static bool taosCheckFileIsOpen(char *logFileName) {
}
}
static void taosGetLogFileName(char *fn) {
static void decideLogFileName(const char *fn, int32_t maxFileNum) {
tsLogObj.fileNum = maxFileNum;
if (tsLogObj.fileNum > 1) {
for (int32_t i = 0; i < tsLogObj.fileNum; i++) {
char fileName[LOG_FILE_NAME_LEN];
@ -438,32 +462,20 @@ static void taosGetLogFileName(char *fn) {
}
}
static int32_t taosOpenLogFile(char *fn, int32_t maxFileNum) {
#ifdef WINDOWS_STASH
/*
* always set maxFileNum to 1
* means client log filename is unique in windows
*/
maxFileNum = 1;
#endif
static void decideLogFileNameFlag(){
char name[LOG_FILE_NAME_LEN + 50] = "\0";
int32_t logstat0_mtime, logstat1_mtime;
int32_t logstat0_mtime = 0;
int32_t logstat1_mtime = 0;
bool log0Exist = false;
bool log1Exist = false;
tsLogObj.fileNum = maxFileNum;
taosGetLogFileName(fn);
if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) {
strcpy(name, fn);
if (strlen(tsLogObj.logName) < LOG_FILE_NAME_LEN + 50 - 2) {
strcpy(name, tsLogObj.logName);
strcat(name, ".0");
log0Exist = taosStatFile(name, NULL, &logstat0_mtime, NULL) >= 0;
name[strlen(name) - 1] = '1';
log1Exist = taosStatFile(name, NULL, &logstat1_mtime, NULL) >= 0;
}
bool log0Exist = taosStatFile(name, NULL, &logstat0_mtime, NULL) >= 0;
if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) {
strcpy(name, fn);
strcat(name, ".1");
}
bool log1Exist = taosStatFile(name, NULL, &logstat1_mtime, NULL) >= 0;
// if none of the log files exist, open 0, if both exists, open the old one
if (!log0Exist && !log1Exist) {
@ -475,16 +487,37 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxFileNum) {
} else {
tsLogObj.flag = (logstat0_mtime > logstat1_mtime) ? 0 : 1;
}
}
char fileName[LOG_FILE_NAME_LEN + 50] = "\0";
sprintf(fileName, "%s.%d", tsLogObj.logName, tsLogObj.flag);
static void processLogFileName(const char* logName , int32_t maxFileNum){
char fullName[PATH_MAX] = {0};
getFullPathName(fullName, logName);
decideLogFileName(fullName, maxFileNum);
decideLogFileNameFlag();
}
static int32_t taosInitNormalLog(const char *logName, int32_t maxFileNum) {
#ifdef WINDOWS_STASH
/*
* always set maxFileNum to 1
* means client log filename is unique in windows
*/
maxFileNum = 1;
#endif
processLogFileName(logName, maxFileNum);
char name[LOG_FILE_NAME_LEN + 50] = "\0";
sprintf(name, "%s.%d", tsLogObj.logName, tsLogObj.flag);
(void)taosThreadMutexInit(&tsLogObj.logMutex, NULL);
(void)taosUmaskFile(0);
tsLogObj.logHandle->pFile = taosOpenFile(fileName, TD_FILE_CREATE | TD_FILE_WRITE);
tsLogObj.logHandle = taosLogBuffNew(LOG_DEFAULT_BUF_SIZE);
if (tsLogObj.logHandle == NULL) return terrno;
tsLogObj.logHandle->pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE);
if (tsLogObj.logHandle->pFile == NULL) {
printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno));
printf("\nfailed to open log file:%s, reason:%s\n", name, strerror(errno));
return TAOS_SYSTEM_ERROR(errno);
}
(void)taosLockLogFile(tsLogObj.logHandle->pFile);
@ -492,7 +525,7 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxFileNum) {
// only an estimate for number of lines
int64_t filesize = 0;
if (taosFStatFile(tsLogObj.logHandle->pFile, &filesize, NULL) < 0) {
printf("\nfailed to fstat log file:%s, reason:%s\n", fileName, strerror(errno));
printf("\nfailed to fstat log file:%s, reason:%s\n", name, strerror(errno));
return TAOS_SYSTEM_ERROR(errno);
}
tsLogObj.lines = (int32_t)(filesize / 60);
@ -610,6 +643,21 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons
taosMemoryFree(buffer);
}
static void backUpSlowLog(){
char buf[64] = {0};
time_t t = taosTime(NULL);
struct tm tmInfo;
if (taosLocalTime(&t, &tmInfo, buf) != NULL) {
(void)strftime(buf, sizeof(buf), "%Y-%m-%d", &tmInfo);
}
// (void)taosThreadMutexLock(&tsLogObj.logMutex);
// if (strlen(tsLogObj.slowLogDay) == 0) {
// tstrncpy(tsLogObj.slowLogDay, buf, 64);
// }else if (strcmp(tsLogObj.slowLogDay, buf) != 0) {
(void)taosOpenNewSlowLogFile(buf);
// }
(void)taosThreadMutexUnlock(&tsLogObj.logMutex);
}
void taosPrintSlowLog(const char *format, ...) {
if (!osLogSpaceAvailable()) return;
@ -834,7 +882,11 @@ static void *taosAsyncOutputLog(void *param) {
int32_t writeInterval = 0;
while (1) {
writeInterval = TMIN(pLogBuf->writeInterval, pSlowBuf->writeInterval);
if (pSlowBuf) {
writeInterval = TMIN(pLogBuf->writeInterval, pSlowBuf->writeInterval);
} else {
writeInterval = pLogBuf->writeInterval;
}
count += writeInterval;
updateCron++;
taosMsleep(writeInterval);
@ -845,17 +897,12 @@ static void *taosAsyncOutputLog(void *param) {
// Polling the buffer
taosWriteLog(pLogBuf);
taosWriteLog(pSlowBuf);
if (pSlowBuf) taosWriteLog(pSlowBuf);
if (updateCron >= 3600 * 24 * 40 / 2) {
taosUpdateDaylight();
updateCron = 0;
}
if (pLogBuf->stop || pSlowBuf->stop) {
if (pLogBuf->stop || (pSlowBuf && pSlowBuf->stop)) {
pLogBuf->lastDuration = LOG_MAX_WAIT_MSEC;
taosWriteLog(pLogBuf);
taosWriteLog(pSlowBuf);
if (pSlowBuf) taosWriteLog(pSlowBuf);
break;
}
}

View File

@ -154,7 +154,7 @@ int main(int argc, char *argv[]) {
}
}
taosInitLog("tref.log", 10);
taosInitLog("tref.log", 10, false);
SRefSpace *pSpaceList = (SRefSpace *)taosMemoryCalloc(sizeof(SRefSpace), threads);
TdThread *pThreadList = (TdThread *)taosMemoryCalloc(sizeof(TdThread), threads);