Merge pull request #3430 from taosdata/feature/crash

Feature/crash

packaging/cfg/taos.cfg

@@ -189,6 +189,9 @@
 # max number of rows per log filters
 # numOfLogLines         10000000
 
+# time of keeping log files, days
+# logKeepDays           0
+
 # enable/disable async log
 # asyncLog              1
 

src/common/inc/tglobal.h

@@ -158,6 +158,7 @@ extern char buildinfo[];
 // log
 extern int32_t tsAsyncLog;
 extern int32_t tsNumOfLogLines;
+extern int32_t tsLogKeepDays;
 extern int32_t dDebugFlag;
 extern int32_t vDebugFlag;
 extern int32_t mDebugFlag;

src/common/src/tglobal.c

@@ -254,7 +254,7 @@ bool taosCfgDynamicOptions(char *msg) {
     //if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue;
     if (cfg->valType != TAOS_CFG_VTYPE_INT32) continue;
     
-    int32_t cfgLen = strlen(cfg->option);
+    int32_t cfgLen = (int32_t)strlen(cfg->option);
     if (cfgLen != olen) continue;
     if (strncasecmp(option, cfg->option, olen) != 0) continue;
     *((int32_t *)cfg->ptr) = vint;
@@ -1013,12 +1013,22 @@ static void doInitGlobalConfig(void) {
   cfg.ptr = &tsNumOfLogLines;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
-  cfg.minValue = 10000;
+  cfg.minValue = 1000;
   cfg.maxValue = 2000000000;
   cfg.ptrLength = 0;
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "logKeepDays";
+  cfg.ptr = &tsLogKeepDays;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
+  cfg.minValue = 0;
+  cfg.maxValue = 365000;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   cfg.option = "asyncLog";
   cfg.ptr = &tsAsyncLog;
   cfg.valType = TAOS_CFG_VTYPE_INT16;

src/os/inc/osDir.h

@@ -24,6 +24,7 @@ extern "C" {
 void taosRemoveDir(char *rootDir);
 int  taosMkDir(const char *pathname, mode_t mode); 
 void taosRename(char* oldName, char *newName);
+void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays);
 
 #ifdef __cplusplus
 }

src/os/inc/osWindows.h

@@ -39,6 +39,7 @@
 #include <Winsock2.h>
 #include <time.h>
 #include <inttypes.h>
+#include <conio.h>
 #include "msvcProcess.h"
 #include "msvcDirect.h"
 #include "msvcFcntl.h"
@@ -58,8 +59,6 @@ extern "C" {
   int32_t BUILDIN_CTZL(uint64_t val);
   int32_t BUILDIN_CTZ(uint32_t val);
 
-#define TAOS_OS_FUNC_DIR
-
 #define TAOS_OS_FUNC_FILE
 #define TAOS_OS_FUNC_FILE_ISREG
 #define TAOS_OS_FUNC_FILE_ISDIR

src/os/src/detail/osDir.c

@@ -18,8 +18,6 @@
 #include "tglobal.h"
 #include "tulog.h"
 
-#ifndef TAOS_OS_FUNC_DIR
-
 void taosRemoveDir(char *rootDir) {
   DIR *dir = opendir(rootDir);
   if (dir == NULL) return;
@@ -51,18 +49,54 @@ int taosMkDir(const char *path, mode_t mode) {
 }
 
 void taosRename(char* oldName, char *newName) {
-  if (0 == tsEnableVnodeBak) {
-    uInfo("vnode backup not enabled");
-    return;
-  }
-
   // if newName in not empty, rename return fail. 
   // the newName must be empty or does not exist
   if (rename(oldName, newName)) {
-    uError("%s is modify to %s fail, reason:%s", oldName, newName, strerror(errno));
+    uError("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
   } else {
-    uInfo("%s is modify to %s success!", oldName, newName);
+    uInfo("successfully to rename file %s to %s", oldName, newName);
   }
 }
 
-#endif
+void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays) {
+  DIR *dir = opendir(rootDir);
+  if (dir == NULL) return;
+
+  int64_t sec = taosGetTimestampSec();
+  struct dirent *de = NULL;
+
+  while ((de = readdir(dir)) != NULL) {
+    if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
+
+    char filename[1024];
+    snprintf(filename, 1023, "%s/%s", rootDir, de->d_name);
+    if (de->d_type & DT_DIR) {
+      continue;
+    } else {
+      // struct stat fState;
+      // if (stat(fname, &fState) < 0) {
+      //   continue;
+      // }
+      int32_t len = (int32_t)strlen(filename);
+      int64_t fileSec = 0;
+      for (int i = len - 1; i >= 0; i--) {
+        if (filename[i] == '.') {
+          fileSec = atoll(filename + i + 1);
+          break;
+        }
+      }
+
+      if (fileSec <= 100) continue;
+      int32_t days = (int32_t)(ABS(sec - fileSec) / 86400 + 1);
+      if (days > keepDays) {
+        (void)remove(filename);
+        uInfo("file:%s is removed, days:%d keepDays:%d", filename, days, keepDays);
+      } else {
+        uTrace("file:%s won't be removed, days:%d keepDays:%d", filename, days, keepDays);
+      }
+    }
+  }
+
+  closedir(dir);
+  rmdir(rootDir);
+}

src/os/src/windows/wDir.c

@@ -1,31 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "os.h"
-#include "tulog.h"
-
-void taosRemoveDir(char *rootDir) {
-  uError("%s not implemented yet", __FUNCTION__);
-}
-
-int taosMkDir(const char *path, mode_t mode) {
-  uError("%s not implemented yet", __FUNCTION__);
-  return 0;
-}
-
-void taosMvDir(char* destDir, char *srcDir) {
-  uError("%s not implemented yet", __FUNCTION__);
-}

src/os/src/windows/wString.c

@@ -58,11 +58,20 @@ char *strsep(char **stringp, const char *delim) {
 
 char *getpass(const char *prefix) {
   static char passwd[TSDB_KEY_LEN] = {0};
-
+  memset(passwd, 0, TSDB_KEY_LEN);
   printf("%s", prefix);
-  scanf("%s", passwd);
 
-  char n = getchar();
+  int32_t index = 0;
+  char    ch;
+  while (index < TSDB_KEY_LEN) {
+    ch = getch();
+    if (ch == '\n' || ch == '\r') {
+      break;
+    } else {
+      passwd[index++] = ch;
+    }
+  }
+
   return passwd;
 }
 
@@ -131,11 +140,11 @@ int tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int bytes) {
 }
 
 
-/* Copy memory to memory until the specified number of bytes
+/* Copy memory to memory until the specified number of bytes
-has been copied, return pointer to following byte.
+has been copied, return pointer to following byte.
-Overlap is NOT handled correctly. */
+Overlap is NOT handled correctly. */
-void *mempcpy(void *dest, const void *src, size_t len) {
+void *mempcpy(void *dest, const void *src, size_t len) {
-  return (char*)memcpy(dest, src, len) + len;
+  return (char*)memcpy(dest, src, len) + len;
 }
 
 /* Copy SRC to DEST, returning the address of the terminating '\0' in DEST.  */

src/tsdb/src/tsdbFile.c

@@ -576,5 +576,5 @@ static TSKEY tsdbGetCurrMinKey(int8_t precision, int32_t keep) {
 }
 
 static int tsdbGetCurrMinFid(int8_t precision, int32_t keep, int32_t days) {
-  return TSDB_KEY_FILEID(tsdbGetCurrMinKey(precision, keep), days, precision);
+  return (int)(TSDB_KEY_FILEID(tsdbGetCurrMinKey(precision, keep), days, precision));
 }

src/util/src/tconfig.c

@@ -270,7 +270,7 @@ void taosReadGlobalLogCfg() {
   }
   wordfree(&full_path);
 
-  taosReadLogOption("tsLogDir", tsLogDir);
+  taosReadLogOption("logDir", tsLogDir);
   
   sprintf(fileName, "%s/taos.cfg", configDir);
   fp = fopen(fileName, "r");
@@ -288,9 +288,9 @@ void taosReadGlobalLogCfg() {
     option = value = NULL;
     olen = vlen = 0;
 
-	taosGetline(&line, &len, fp);
+    taosGetline(&line, &len, fp);
     line[len - 1] = 0;
-    
+
     paGetToken(line, &option, &olen);
     if (olen == 0) continue;
     option[olen] = 0;

src/util/src/tlog.c

@@ -62,6 +62,7 @@ typedef struct {
   pthread_mutex_t logMutex;
 } SLogObj;
 
+int32_t tsLogKeepDays = 0;
 int32_t tsAsyncLog = 1;
 float   tsTotalLogDirGB = 0;
 float   tsAvailLogDirGB = 0;
@@ -78,6 +79,7 @@ static int32_t   taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen
 static SLogBuff *taosLogBuffNew(int32_t bufSize);
 static void      taosCloseLogByFd(int32_t oldFd);
 static int32_t   taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum);
+extern void      taosPrintGlobalCfg();
 
 static int32_t taosStartLog() {
   pthread_attr_t threadAttr;
@@ -136,11 +138,24 @@ static void taosUnLockFile(int32_t fd) {
   }
 }
 
+static void taosKeepOldLog(char *oldName) {
+  if (tsLogKeepDays <= 0) return;
+
+  int64_t fileSec = taosGetTimestampSec();
+  char    fileName[LOG_FILE_NAME_LEN + 20];
+  snprintf(fileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64, tsLogObj.logName, fileSec);
+
+  taosRename(oldName, fileName);
+  taosRemoveOldLogFiles(tsLogDir, tsLogKeepDays);
+}
+
 static void *taosThreadToOpenNewFile(void *param) {
-  char name[LOG_FILE_NAME_LEN + 20];
+  char keepName[LOG_FILE_NAME_LEN + 20];
+  sprintf(keepName, "%s.%d", tsLogObj.logName, tsLogObj.flag);
 
   tsLogObj.flag ^= 1;
   tsLogObj.lines = 0;
+  char name[LOG_FILE_NAME_LEN + 20];
   sprintf(name, "%s.%d", tsLogObj.logName, tsLogObj.flag);
 
   umask(0);
@@ -150,6 +165,7 @@ static void *taosThreadToOpenNewFile(void *param) {
     uError("open new log file fail! fd:%d reason:%s", fd, strerror(errno));
     return NULL;
   }
+
   taosLockFile(fd);
   (void)lseek(fd, 0, SEEK_SET);
 
@@ -157,9 +173,13 @@ static void *taosThreadToOpenNewFile(void *param) {
   tsLogObj.logHandle->fd = fd;
   tsLogObj.lines = 0;
   tsLogObj.openInProgress = 0;
-  uInfo("new log file is opened!!!");
-
   taosCloseLogByFd(oldFd);
+  
+  uInfo("   new log file:%d is opened", tsLogObj.flag);
+  uInfo("==================================");
+  taosPrintGlobalCfg();
+  taosKeepOldLog(keepName);
+
   return NULL;
 }
 
@@ -264,20 +284,23 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
     strcat(name, ".0");
   }
 
+  if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) {
+    strcpy(name, fn);
+    strcat(name, ".1");
+  }
+
+  bool log0Exist = stat(name, &logstat0) >= 0;
+  bool log1Exist = stat(name, &logstat1) >= 0;
+  
   // if none of the log files exist, open 0, if both exists, open the old one
-  if (stat(name, &logstat0) < 0) {
+  if (!log0Exist && !log1Exist) {
     tsLogObj.flag = 0;
+  } else if (!log1Exist) {
+    tsLogObj.flag = 0;
+  } else if (!log0Exist) {
+    tsLogObj.flag = 1;
   } else {
-    if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) {
+    tsLogObj.flag = (logstat0.st_mtime > logstat1.st_mtime) ? 0 : 1;
-      strcpy(name, fn);
-      strcat(name, ".1");
-    }
-    
-    if (stat(name, &logstat1) < 0) {
-      tsLogObj.flag = 1;
-    } else {
-      tsLogObj.flag = (logstat0.st_mtime > logstat1.st_mtime) ? 0 : 1;
-    }
   }
 
   char fileName[LOG_FILE_NAME_LEN + 50] = "\0";

src/vnode/src/vnodeMain.c

@@ -382,7 +382,13 @@ void vnodeRelease(void *pVnodeRaw) {
     char newDir[TSDB_FILENAME_LEN] = {0};
     sprintf(rootDir, "%s/vnode%d", tsVnodeDir, vgId);
     sprintf(newDir, "%s/vnode%d", tsVnodeBakDir, vgId);
-    taosRename(rootDir, newDir);
+
+    if (0 == tsEnableVnodeBak) {
+      vInfo("vgId:%d, vnode backup not enabled", pVnode->vgId);
+    } else {
+      taosRename(rootDir, newDir);
+    }
+
     taosRemoveDir(rootDir);
     dnodeSendStatusMsgToMnode();
   }