Merge pull request #29016 from taosdata/enh/TD-30933-3.0
enh(log): specify log output and process log rotate periodically
This commit is contained in:
Shengliang Guan
GitHub
commit
178a95542a
|
|
@ -44,7 +44,7 @@ typedef struct {
|
|||
|
||||
char file_path[TSDB_FILENAME_LEN]; // local file path
|
||||
int64_t file_size; // local file size, for upload
|
||||
int32_t file_last_modified; // local file last modified time, for upload
|
||||
int64_t file_last_modified; // local file last modified time, for upload
|
||||
char file_md5[64]; // md5 of the local file content, for upload, reserved
|
||||
|
||||
char object_name[128]; // object name
|
||||
|
|
@ -67,9 +67,9 @@ int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint);
|
|||
int32_t cos_cp_dump(SCheckpoint* checkpoint);
|
||||
void cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPart* parts, int64_t* consume_bytes);
|
||||
void cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag, uint64_t crc64);
|
||||
void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int32_t mtime,
|
||||
void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int64_t mtime,
|
||||
char const* upload_id, int64_t part_size);
|
||||
bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int32_t mtime);
|
||||
bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int64_t mtime);
|
||||
|
||||
void cos_cp_build_download(SCheckpoint* checkpoint, char const* filepath, char const* object_name, int64_t object_size,
|
||||
char const* object_lmtime, char const* object_etag, int64_t part_size);
|
||||
|
|
|
|||
|
|
@ -82,9 +82,9 @@ int32_t taosUnLockFile(TdFilePtr pFile);
|
|||
|
||||
int32_t taosUmaskFile(int32_t maskVal);
|
||||
|
||||
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *atime);
|
||||
int32_t taosStatFile(const char *path, int64_t *size, int64_t *mtime, int64_t *atime);
|
||||
int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno);
|
||||
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime);
|
||||
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int64_t *mtime);
|
||||
bool taosCheckExistFile(const char *pathname);
|
||||
|
||||
int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence);
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ extern bool tsAsyncLog;
|
|||
extern bool tsAssert;
|
||||
extern int32_t tsNumOfLogLines;
|
||||
extern int32_t tsLogKeepDays;
|
||||
extern char *tsLogOutput;
|
||||
extern LogFp tsLogFp;
|
||||
extern int64_t tsNumOfErrorLogs;
|
||||
extern int64_t tsNumOfInfoLogs;
|
||||
|
|
@ -71,7 +72,7 @@ extern int32_t sndDebugFlag;
|
|||
extern int32_t simDebugFlag;
|
||||
|
||||
extern int32_t tqClientDebugFlag;
|
||||
|
||||
int32_t taosInitLogOutput(const char **ppLogName);
|
||||
int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc);
|
||||
void taosCloseLog();
|
||||
void taosResetLog();
|
||||
|
|
|
|||
|
|
@ -56,6 +56,8 @@ void taosIpPort2String(uint32_t ip, uint16_t port, char *str);
|
|||
void *tmemmem(const char *haystack, int hlen, const char *needle, int nlen);
|
||||
|
||||
int32_t parseCfgReal(const char *str, float *out);
|
||||
bool tIsValidFileName(const char *fileName, const char *pattern);
|
||||
bool tIsValidFilePath(const char *filePath, const char *pattern);
|
||||
|
||||
static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) {
|
||||
T_MD5_CTX context;
|
||||
|
|
|
|||
|
|
@ -41,6 +41,10 @@
|
|||
#include "cus_name.h"
|
||||
#endif
|
||||
|
||||
#ifndef CUS_PROMPT
|
||||
#define CUS_PROMPT "tao"
|
||||
#endif
|
||||
|
||||
#define TSC_VAR_NOT_RELEASE 1
|
||||
#define TSC_VAR_RELEASED 0
|
||||
|
||||
|
|
@ -954,14 +958,10 @@ void taos_init_imp(void) {
|
|||
taosHashSetFreeFp(appInfo.pInstMap, destroyAppInst);
|
||||
deltaToUtcInitOnce();
|
||||
|
||||
char logDirName[64] = {0};
|
||||
#ifdef CUS_PROMPT
|
||||
snprintf(logDirName, 64, "%slog", CUS_PROMPT);
|
||||
#else
|
||||
(void)snprintf(logDirName, 64, "taoslog");
|
||||
#endif
|
||||
if (taosCreateLog(logDirName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) {
|
||||
(void)printf(" WARING: Create %s failed:%s. configDir=%s\n", logDirName, strerror(errno), configDir);
|
||||
const char *logName = CUS_PROMPT "slog";
|
||||
ENV_ERR_RET(taosInitLogOutput(&logName), "failed to init log output");
|
||||
if (taosCreateLog(logName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) {
|
||||
(void)printf(" WARING: Create %s failed:%s. configDir=%s\n", logName, strerror(errno), configDir);
|
||||
tscInitRes = -1;
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -775,7 +775,7 @@ _exit:
|
|||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const char *file, int32_t lmtime,
|
||||
static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const char *file, int64_t lmtime,
|
||||
char const *object_name, int64_t contentLength, S3PutProperties *put_prop,
|
||||
put_object_callback_data *data) {
|
||||
int32_t code = 0, lino = 0;
|
||||
|
|
@ -963,7 +963,7 @@ _exit:
|
|||
|
||||
int32_t s3PutObjectFromFile2ByEp(const char *file, const char *object_name, int8_t withcp, int8_t epIndex) {
|
||||
int32_t code = 0;
|
||||
int32_t lmtime = 0;
|
||||
int64_t lmtime = 0;
|
||||
const char *filename = 0;
|
||||
uint64_t contentLength = 0;
|
||||
const char *cacheControl = 0, *contentType = 0, *md5 = 0;
|
||||
|
|
@ -1040,7 +1040,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t w
|
|||
static int32_t s3PutObjectFromFileOffsetByEp(const char *file, const char *object_name, int64_t offset, int64_t size,
|
||||
int8_t epIndex) {
|
||||
int32_t code = 0;
|
||||
int32_t lmtime = 0;
|
||||
int64_t lmtime = 0;
|
||||
const char *filename = 0;
|
||||
uint64_t contentLength = 0;
|
||||
const char *cacheControl = 0, *contentType = 0, *md5 = 0;
|
||||
|
|
@ -1847,7 +1847,7 @@ _exit:
|
|||
|
||||
typedef struct {
|
||||
int64_t size;
|
||||
int32_t atime;
|
||||
int64_t atime;
|
||||
char name[TSDB_FILENAME_LEN];
|
||||
} SEvictFile;
|
||||
|
||||
|
|
|
|||
|
|
@ -350,7 +350,7 @@ void cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag
|
|||
checkpoint->parts[part_index].crc64 = crc64;
|
||||
}
|
||||
|
||||
void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int32_t mtime,
|
||||
void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int64_t mtime,
|
||||
char const* upload_id, int64_t part_size) {
|
||||
int i = 0;
|
||||
|
||||
|
|
@ -375,7 +375,7 @@ void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t
|
|||
|
||||
static bool cos_cp_verify_md5(SCheckpoint* cp) { return true; }
|
||||
|
||||
bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int32_t mtime) {
|
||||
bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int64_t mtime) {
|
||||
if (cos_cp_verify_md5(checkpoint) && checkpoint->file_size == size && checkpoint->file_last_modified == mtime) {
|
||||
return true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
#include "tglobal.h"
|
||||
#include "defines.h"
|
||||
#include "os.h"
|
||||
#include "osString.h"
|
||||
#include "tconfig.h"
|
||||
#include "tgrant.h"
|
||||
#include "tlog.h"
|
||||
|
|
@ -1010,12 +1011,43 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
|
|||
TAOS_RETURN(TSDB_CODE_SUCCESS);
|
||||
}
|
||||
|
||||
static int32_t taosSetLogOutput(SConfig *pCfg) {
|
||||
if (tsLogOutput) {
|
||||
char *pLog = tsLogOutput;
|
||||
char *pEnd = NULL;
|
||||
if (strcasecmp(pLog, "stdout") && strcasecmp(pLog, "stderr") && strcasecmp(pLog, "/dev/null")) {
|
||||
if ((pEnd = strrchr(pLog, '/')) || (pEnd = strrchr(pLog, '\\'))) {
|
||||
int32_t pathLen = POINTER_DISTANCE(pEnd, pLog) + 1;
|
||||
if (*pLog == '/' || *pLog == '\\') {
|
||||
if (pathLen <= 0 || pathLen > PATH_MAX) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
|
||||
tstrncpy(tsLogDir, pLog, pathLen);
|
||||
} else {
|
||||
int32_t len = strlen(tsLogDir);
|
||||
if (len < 0 || len >= (PATH_MAX - 1)) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
|
||||
if (len == 0 || (tsLogDir[len - 1] != '/' && tsLogDir[len - 1] != '\\')) {
|
||||
tsLogDir[len++] = TD_DIRSEP_CHAR;
|
||||
}
|
||||
int32_t remain = PATH_MAX - len - 1;
|
||||
if (remain < pathLen) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
|
||||
tstrncpy(tsLogDir + len, pLog, pathLen);
|
||||
}
|
||||
TAOS_CHECK_RETURN(cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true));
|
||||
}
|
||||
} else {
|
||||
tstrncpy(tsLogDir, pLog, PATH_MAX);
|
||||
TAOS_CHECK_RETURN(cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true));
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t taosSetClientLogCfg(SConfig *pCfg) {
|
||||
SConfigItem *pItem = NULL;
|
||||
|
||||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "logDir");
|
||||
tstrncpy(tsLogDir, pItem->str, PATH_MAX);
|
||||
TAOS_CHECK_RETURN(taosExpandDir(tsLogDir, tsLogDir, PATH_MAX));
|
||||
TAOS_CHECK_RETURN(taosSetLogOutput(pCfg));
|
||||
|
||||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "minimalLogDirGB");
|
||||
tsLogSpace.reserved = (int64_t)(((double)pItem->fval) * 1024 * 1024 * 1024);
|
||||
|
|
@ -1860,8 +1892,10 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
|
|||
}
|
||||
|
||||
if (tsc) {
|
||||
TAOS_CHECK_GOTO(taosSetClientLogCfg(tsCfg), &lino, _exit);
|
||||
TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit);
|
||||
} else {
|
||||
TAOS_CHECK_GOTO(taosSetClientLogCfg(tsCfg), &lino, _exit);
|
||||
TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit);
|
||||
TAOS_CHECK_GOTO(taosUpdateServerCfg(tsCfg), &lino, _exit);
|
||||
TAOS_CHECK_GOTO(taosSetServerCfg(tsCfg), &lino, _exit);
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@
|
|||
#define DM_ENV_CMD "The env cmd variable string to use when configuring the server, such as: -e 'TAOS_FQDN=td1'."
|
||||
#define DM_ENV_FILE "The env variable file path to use when configuring the server, default is './.env', .env text can be 'TAOS_FQDN=td1'."
|
||||
#define DM_MACHINE_CODE "Get machine code."
|
||||
#define DM_LOG_OUTPUT "Specify log output. Options:\n\r\t\t\t stdout, stderr, /dev/null, <directory>, <directory>/<filename>, <filename>\n\r\t\t\t * If OUTPUT contains an absolute directory, logs will be stored in that directory instead of logDir.\n\r\t\t\t * If OUTPUT contains a relative directory, logs will be stored in the directory combined with logDir and the relative directory."
|
||||
#define DM_VERSION "Print program version."
|
||||
#define DM_EMAIL "<support@taosdata.com>"
|
||||
#define DM_MEM_DBG "Enable memory debug"
|
||||
|
|
@ -239,6 +240,32 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
|
|||
}
|
||||
} else if (strcmp(argv[i], "-k") == 0) {
|
||||
global.generateGrant = true;
|
||||
#if defined(LINUX)
|
||||
} else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--log-output") == 0 ||
|
||||
strncmp(argv[i], "--log-output=", 13) == 0) {
|
||||
if ((i < argc - 1) || ((i == argc - 1) && strncmp(argv[i], "--log-output=", 13) == 0)) {
|
||||
int32_t klen = strlen(argv[i]);
|
||||
int32_t vlen = klen < 13 ? strlen(argv[++i]) : klen - 13;
|
||||
const char *val = argv[i];
|
||||
if (klen >= 13) val += 13;
|
||||
if (vlen <= 0 || vlen >= PATH_MAX) {
|
||||
printf("failed to set log output since invalid vlen:%d, valid range: [1, %d)\n", vlen, PATH_MAX);
|
||||
return TSDB_CODE_INVALID_CFG;
|
||||
}
|
||||
tsLogOutput = taosMemoryMalloc(PATH_MAX);
|
||||
if (!tsLogOutput) {
|
||||
printf("failed to set log output: '%s' since %s\n", val, tstrerror(terrno));
|
||||
return terrno;
|
||||
}
|
||||
if (taosExpandDir(val, tsLogOutput, PATH_MAX) != 0) {
|
||||
printf("failed to expand log output: '%s' since %s\n", val, tstrerror(terrno));
|
||||
return terrno;
|
||||
}
|
||||
} else {
|
||||
printf("'%s' requires a parameter\n", argv[i]);
|
||||
return TSDB_CODE_INVALID_CFG;
|
||||
}
|
||||
#endif
|
||||
} else if (strcmp(argv[i], "-y") == 0) {
|
||||
global.generateCode = true;
|
||||
if (i < argc - 1) {
|
||||
|
|
@ -316,6 +343,9 @@ static void dmPrintHelp() {
|
|||
printf("%s%s%s%s\n", indent, "-e,", indent, DM_ENV_CMD);
|
||||
printf("%s%s%s%s\n", indent, "-E,", indent, DM_ENV_FILE);
|
||||
printf("%s%s%s%s\n", indent, "-k,", indent, DM_MACHINE_CODE);
|
||||
#if defined(LINUX)
|
||||
printf("%s%s%s%s\n", indent, "-o, --log-output=OUTPUT", indent, DM_LOG_OUTPUT);
|
||||
#endif
|
||||
printf("%s%s%s%s\n", indent, "-y,", indent, DM_SET_ENCRYPTKEY);
|
||||
printf("%s%s%s%s\n", indent, "-dm,", indent, DM_MEM_DBG);
|
||||
printf("%s%s%s%s\n", indent, "-V,", indent, DM_VERSION);
|
||||
|
|
@ -340,8 +370,11 @@ static int32_t dmCheckS3() {
|
|||
}
|
||||
|
||||
static int32_t dmInitLog() {
|
||||
return taosCreateLog(CUS_PROMPT "dlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs,
|
||||
0);
|
||||
const char *logName = CUS_PROMPT "dlog";
|
||||
|
||||
TAOS_CHECK_RETURN(taosInitLogOutput(&logName));
|
||||
|
||||
return taosCreateLog(logName, 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0);
|
||||
}
|
||||
|
||||
static void taosCleanupArgs() {
|
||||
|
|
|
|||
|
|
@ -660,7 +660,7 @@ static int32_t tsdbDoS3Migrate(SRTNer *rtner) {
|
|||
int32_t lcn = fobj->f->lcn;
|
||||
|
||||
if (/*lcn < 1 && */ taosCheckExistFile(fobj->fname)) {
|
||||
int32_t mtime = 0;
|
||||
int64_t mtime = 0;
|
||||
int64_t size = 0;
|
||||
int32_t r = taosStatFile(fobj->fname, &size, &mtime, NULL);
|
||||
if (size > chunksize && mtime < rtner->now - tsS3UploadDelaySec) {
|
||||
|
|
@ -687,7 +687,7 @@ static int32_t tsdbDoS3Migrate(SRTNer *rtner) {
|
|||
tsdbTFileLastChunkName(rtner->tsdb, fobj->f, fname1);
|
||||
|
||||
if (taosCheckExistFile(fname1)) {
|
||||
int32_t mtime = 0;
|
||||
int64_t mtime = 0;
|
||||
int64_t size = 0;
|
||||
if (taosStatFile(fname1, &size, &mtime, NULL) != 0) {
|
||||
tsdbError("vgId:%d, %s failed at %s:%d ", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, __LINE__);
|
||||
|
|
|
|||
|
|
@ -1564,8 +1564,8 @@ static void udfdPrintVersion() {
|
|||
}
|
||||
|
||||
static int32_t udfdInitLog() {
|
||||
char logName[12] = {0};
|
||||
snprintf(logName, sizeof(logName), "%slog", "udfd");
|
||||
const char *logName = "udfdlog";
|
||||
TAOS_CHECK_RETURN(taosInitLogOutput(&logName));
|
||||
return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, NULL, 0);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -326,7 +326,7 @@ static int32_t walRepairLogFileTs(SWal* pWal, bool* updateMeta) {
|
|||
}
|
||||
|
||||
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
|
||||
int32_t mtime = 0;
|
||||
int64_t mtime = 0;
|
||||
if (taosStatFile(fnameStr, NULL, &mtime, NULL) < 0) {
|
||||
wError("vgId:%d, failed to stat file due to %s, file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
|
||||
|
||||
|
|
|
|||
|
|
@ -303,7 +303,7 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) {
|
|||
if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue;
|
||||
|
||||
char filename[1024];
|
||||
(void)snprintf(filename, sizeof(filename), "%s/%s", dirname, taosGetDirEntryName(de));
|
||||
(void)snprintf(filename, sizeof(filename), "%s%s%s", dirname, TD_DIRSEP, taosGetDirEntryName(de));
|
||||
if (taosDirEntryIsDir(de)) {
|
||||
continue;
|
||||
} else {
|
||||
|
|
@ -349,7 +349,7 @@ int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) {
|
|||
wordfree(&full_path);
|
||||
// FALL THROUGH
|
||||
default:
|
||||
return code;
|
||||
return terrno = TSDB_CODE_INVALID_PARA;
|
||||
}
|
||||
|
||||
if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) {
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ int32_t osDefaultInit() {
|
|||
}
|
||||
tstrncpy(tsDataDir, TD_DATA_DIR_PATH, sizeof(tsDataDir));
|
||||
tstrncpy(tsLogDir, TD_LOG_DIR_PATH, sizeof(tsLogDir));
|
||||
if (strlen(tsTempDir) == 0){
|
||||
if (strlen(tsTempDir) == 0) {
|
||||
tstrncpy(tsTempDir, TD_TMP_DIR_PATH, sizeof(tsTempDir));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -273,7 +273,7 @@ int32_t taosRenameFile(const char *oldName, const char *newName) {
|
|||
#endif
|
||||
}
|
||||
|
||||
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *atime) {
|
||||
int32_t taosStatFile(const char *path, int64_t *size, int64_t *mtime, int64_t *atime) {
|
||||
OS_PARAM_CHECK(path);
|
||||
#ifdef WINDOWS
|
||||
struct _stati64 fileStat;
|
||||
|
|
@ -544,7 +544,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) {
|
|||
return liOffset.QuadPart;
|
||||
}
|
||||
|
||||
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) {
|
||||
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int64_t *mtime) {
|
||||
if (pFile == NULL || pFile->hFile == NULL) {
|
||||
terrno = TSDB_CODE_INVALID_PARA;
|
||||
return terrno;
|
||||
|
|
@ -571,7 +571,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) {
|
|||
ULARGE_INTEGER ull;
|
||||
ull.LowPart = lastWriteTime.dwLowDateTime;
|
||||
ull.HighPart = lastWriteTime.dwHighDateTime;
|
||||
*mtime = (int32_t)((ull.QuadPart - 116444736000000000ULL) / 10000000ULL);
|
||||
*mtime = (int64_t)((ull.QuadPart - 116444736000000000ULL) / 10000000ULL);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -937,7 +937,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) {
|
|||
return ret;
|
||||
}
|
||||
|
||||
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) {
|
||||
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int64_t *mtime) {
|
||||
if (pFile == NULL) {
|
||||
terrno = TSDB_CODE_INVALID_PARA;
|
||||
return terrno;
|
||||
|
|
|
|||
|
|
@ -51,6 +51,13 @@
|
|||
#define LOG_EDITION_FLG ("C")
|
||||
#endif
|
||||
|
||||
typedef enum {
|
||||
LOG_OUTPUT_FILE = 0, // default
|
||||
LOG_OUTPUT_STDOUT = 1, // stdout set by -o option on the command line
|
||||
LOG_OUTPUT_STDERR = 2, // stderr set by -o option on the command line
|
||||
LOG_OUTPUT_NULL = 4, // /dev/null set by -o option on the command line
|
||||
} ELogOutputType;
|
||||
|
||||
typedef struct {
|
||||
char *buffer;
|
||||
int32_t buffStart;
|
||||
|
|
@ -73,6 +80,7 @@ typedef struct {
|
|||
int32_t openInProgress;
|
||||
int64_t lastKeepFileSec;
|
||||
int64_t timestampToday;
|
||||
int8_t outputType; // ELogOutputType
|
||||
pid_t pid;
|
||||
char logName[PATH_MAX];
|
||||
char slowLogName[PATH_MAX];
|
||||
|
|
@ -86,6 +94,7 @@ static int8_t tsLogInited = 0;
|
|||
static SLogObj tsLogObj = {.fileNum = 1, .slowHandle = NULL};
|
||||
static int64_t tsAsyncLogLostLines = 0;
|
||||
static int32_t tsDaylightActive; /* Currently in daylight saving time. */
|
||||
static SRWLatch tsLogRotateLatch = 0;
|
||||
|
||||
bool tsLogEmbedded = 0;
|
||||
bool tsAsyncLog = true;
|
||||
|
|
@ -96,6 +105,7 @@ bool tsAssert = true;
|
|||
#endif
|
||||
int32_t tsNumOfLogLines = 10000000;
|
||||
int32_t tsLogKeepDays = 0;
|
||||
char *tsLogOutput = NULL;
|
||||
LogFp tsLogFp = NULL;
|
||||
int64_t tsNumOfErrorLogs = 0;
|
||||
int64_t tsNumOfInfoLogs = 0;
|
||||
|
|
@ -234,6 +244,58 @@ int32_t taosInitSlowLog() {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t taosInitLogOutput(const char **ppLogName) {
|
||||
const char *pLog = tsLogOutput;
|
||||
const char *pLogName = NULL;
|
||||
if (pLog) {
|
||||
if (!tIsValidFilePath(pLog, NULL)) {
|
||||
fprintf(stderr, "invalid log output destination:%s, contains illegal char\n", pLog);
|
||||
return TSDB_CODE_INVALID_CFG;
|
||||
}
|
||||
if (0 == strcasecmp(pLog, "stdout")) {
|
||||
tsLogObj.outputType = LOG_OUTPUT_STDOUT;
|
||||
if (ppLogName) *ppLogName = pLog;
|
||||
return 0;
|
||||
}
|
||||
if (0 == strcasecmp(pLog, "stderr")) {
|
||||
tsLogObj.outputType = LOG_OUTPUT_STDERR;
|
||||
if (ppLogName) *ppLogName = pLog;
|
||||
return 0;
|
||||
}
|
||||
if (0 == strcasecmp(pLog, "/dev/null")) {
|
||||
tsLogObj.outputType = LOG_OUTPUT_NULL;
|
||||
if (ppLogName) *ppLogName = pLog;
|
||||
return 0;
|
||||
}
|
||||
int32_t len = strlen(pLog);
|
||||
if (len < 1) {
|
||||
fprintf(stderr, "invalid log output destination:%s, should not be empty\n", pLog);
|
||||
return TSDB_CODE_INVALID_CFG;
|
||||
}
|
||||
const char *p = pLog + (len - 1);
|
||||
if (*p == '/' || *p == '\\') {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if ((p = strrchr(pLog, '/')) || (p = strrchr(pLog, '\\'))) {
|
||||
pLogName = p + 1;
|
||||
} else {
|
||||
pLogName = pLog;
|
||||
}
|
||||
if (strcmp(pLogName, ".") == 0 || strcmp(pLogName, "..") == 0) {
|
||||
fprintf(stderr, "invalid log output destination:%s\n", pLog);
|
||||
return TSDB_CODE_INVALID_CFG;
|
||||
}
|
||||
|
||||
if (!tIsValidFileName(pLogName, NULL)) {
|
||||
fprintf(stderr, "invalid log output destination:%s, contains illegal char\n", pLog);
|
||||
return TSDB_CODE_INVALID_CFG;
|
||||
}
|
||||
if (ppLogName) *ppLogName = pLogName;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) {
|
||||
if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0;
|
||||
int32_t code = osUpdate();
|
||||
|
|
@ -241,6 +303,11 @@ int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) {
|
|||
uError("failed to update os info, reason:%s", tstrerror(code));
|
||||
}
|
||||
|
||||
if (tsLogObj.outputType == LOG_OUTPUT_STDOUT || tsLogObj.outputType == LOG_OUTPUT_STDERR ||
|
||||
tsLogObj.outputType == LOG_OUTPUT_NULL) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
TAOS_CHECK_RETURN(taosInitNormalLog(logName, maxFiles));
|
||||
if (tsc) {
|
||||
TAOS_CHECK_RETURN(taosInitSlowLog());
|
||||
|
|
@ -283,6 +350,7 @@ void taosCloseLog() {
|
|||
taosMemoryFreeClear(tsLogObj.logHandle);
|
||||
tsLogObj.logHandle = NULL;
|
||||
}
|
||||
taosMemoryFreeClear(tsLogOutput);
|
||||
}
|
||||
|
||||
static bool taosLockLogFile(TdFilePtr pFile) {
|
||||
|
|
@ -340,10 +408,6 @@ static void taosKeepOldLog(char *oldName) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (tsLogKeepDays > 0) {
|
||||
taosRemoveOldFiles(tsLogDir, tsLogKeepDays);
|
||||
}
|
||||
}
|
||||
typedef struct {
|
||||
TdFilePtr pOldFile;
|
||||
|
|
@ -392,11 +456,16 @@ static OldFileKeeper *taosOpenNewFile() {
|
|||
|
||||
static void *taosThreadToCloseOldFile(void *param) {
|
||||
if (!param) return NULL;
|
||||
taosWLockLatch(&tsLogRotateLatch);
|
||||
OldFileKeeper *oldFileKeeper = (OldFileKeeper *)param;
|
||||
taosSsleep(20);
|
||||
taosCloseLogByFd(oldFileKeeper->pOldFile);
|
||||
taosKeepOldLog(oldFileKeeper->keepName);
|
||||
taosMemoryFree(oldFileKeeper);
|
||||
if (tsLogKeepDays > 0) {
|
||||
taosRemoveOldFiles(tsLogDir, tsLogKeepDays);
|
||||
}
|
||||
taosWUnLockLatch(&tsLogRotateLatch);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
|
@ -532,8 +601,8 @@ static void decideLogFileName(const char *fn, int32_t maxFileNum) {
|
|||
|
||||
static void decideLogFileNameFlag() {
|
||||
char name[PATH_MAX + 50] = "\0";
|
||||
int32_t logstat0_mtime = 0;
|
||||
int32_t logstat1_mtime = 0;
|
||||
int64_t logstat0_mtime = 0;
|
||||
int64_t logstat1_mtime = 0;
|
||||
bool log0Exist = false;
|
||||
bool log1Exist = false;
|
||||
|
||||
|
|
@ -680,10 +749,19 @@ static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *b
|
|||
}
|
||||
}
|
||||
|
||||
if (dflag & DEBUG_SCREEN) {
|
||||
int fd = 0;
|
||||
if (tsLogObj.outputType == LOG_OUTPUT_FILE) {
|
||||
if (dflag & DEBUG_SCREEN) fd = 1;
|
||||
} else if (tsLogObj.outputType == LOG_OUTPUT_STDOUT) {
|
||||
fd = 1;
|
||||
} else if (tsLogObj.outputType == LOG_OUTPUT_STDERR) {
|
||||
fd = 2;
|
||||
}
|
||||
|
||||
if (fd) {
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wunused-result"
|
||||
if (write(1, buffer, (uint32_t)len) < 0) {
|
||||
if (write(fd, buffer, (uint32_t)len) < 0) {
|
||||
TAOS_UNUSED(printf("failed to write log to screen, reason:%s\n", strerror(errno)));
|
||||
}
|
||||
#pragma GCC diagnostic pop
|
||||
|
|
@ -999,6 +1077,92 @@ static void taosWriteLog(SLogBuff *pLogBuf) {
|
|||
pLogBuf->writeInterval = 0;
|
||||
}
|
||||
|
||||
#define LOG_ROTATE_INTERVAL 3600
|
||||
#if !defined(TD_ENTERPRISE) || defined(ASSERT_NOT_CORE)
|
||||
#define LOG_INACTIVE_TIME 7200
|
||||
#define LOG_ROTATE_BOOT 900
|
||||
#else
|
||||
#define LOG_INACTIVE_TIME 5
|
||||
#define LOG_ROTATE_BOOT (LOG_INACTIVE_TIME + 1)
|
||||
#endif
|
||||
|
||||
static void *taosLogRotateFunc(void *param) {
|
||||
setThreadName("logRotate");
|
||||
int32_t code = 0;
|
||||
taosWLockLatch(&tsLogRotateLatch);
|
||||
// compress or remove the old log files
|
||||
TdDirPtr pDir = taosOpenDir(tsLogDir);
|
||||
if (!pDir) goto _exit;
|
||||
TdDirEntryPtr de = NULL;
|
||||
while ((de = taosReadDir(pDir))) {
|
||||
if (taosDirEntryIsDir(de)) {
|
||||
continue;
|
||||
}
|
||||
char *fname = taosGetDirEntryName(de);
|
||||
if (!fname) {
|
||||
continue;
|
||||
}
|
||||
|
||||
char *pSec = strrchr(fname, '.');
|
||||
if (!pSec) {
|
||||
continue;
|
||||
}
|
||||
char *pIter = pSec;
|
||||
bool isSec = true;
|
||||
while (*(++pIter)) {
|
||||
if (!isdigit(*pIter)) {
|
||||
isSec = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!isSec) {
|
||||
continue;
|
||||
}
|
||||
|
||||
int64_t fileSec = 0;
|
||||
if ((code = taosStr2int64(pSec + 1, &fileSec)) != 0) {
|
||||
uWarn("%s:%d failed to convert %s to int64 since %s", __func__, __LINE__, pSec + 1, tstrerror(code));
|
||||
continue;
|
||||
}
|
||||
if (fileSec <= 100) {
|
||||
continue;
|
||||
}
|
||||
|
||||
char fullName[PATH_MAX] = {0};
|
||||
snprintf(fullName, sizeof(fullName), "%s%s%s", tsLogDir, TD_DIRSEP, fname);
|
||||
|
||||
int64_t mtime = 0;
|
||||
if ((code = taosStatFile(fullName, NULL, &mtime, NULL)) != 0) {
|
||||
uWarn("%s:%d failed to stat file %s since %s", __func__, __LINE__, fullName, tstrerror(code));
|
||||
continue;
|
||||
}
|
||||
|
||||
int64_t inactiveSec = taosGetTimestampMs() / 1000 - mtime;
|
||||
|
||||
if (inactiveSec < LOG_INACTIVE_TIME) {
|
||||
continue;
|
||||
}
|
||||
|
||||
int32_t days = inactiveSec / 86400 + 1;
|
||||
if (tsLogKeepDays > 0 && days > tsLogKeepDays) {
|
||||
TAOS_UNUSED(taosRemoveFile(fullName));
|
||||
uInfo("file:%s is removed, days:%d, keepDays:%d, sed:%" PRId64, fullName, days, tsLogKeepDays, fileSec);
|
||||
} else {
|
||||
taosKeepOldLog(fullName); // compress
|
||||
}
|
||||
}
|
||||
if ((code = taosCloseDir(&pDir)) != 0) {
|
||||
uWarn("%s:%d failed to close dir %s since %s\n", __func__, __LINE__, tsLogDir, tstrerror(code));
|
||||
}
|
||||
|
||||
if (tsLogKeepDays > 0) {
|
||||
taosRemoveOldFiles(tsLogDir, tsLogKeepDays);
|
||||
}
|
||||
_exit:
|
||||
taosWUnLockLatch(&tsLogRotateLatch);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void *taosAsyncOutputLog(void *param) {
|
||||
SLogBuff *pLogBuf = (SLogBuff *)tsLogObj.logHandle;
|
||||
SLogBuff *pSlowBuf = (SLogBuff *)tsLogObj.slowHandle;
|
||||
|
|
@ -1007,6 +1171,7 @@ static void *taosAsyncOutputLog(void *param) {
|
|||
int32_t count = 0;
|
||||
int32_t updateCron = 0;
|
||||
int32_t writeInterval = 0;
|
||||
int64_t lastCheckSec = taosGetTimestampMs() / 1000 - (LOG_ROTATE_INTERVAL - LOG_ROTATE_BOOT);
|
||||
|
||||
while (1) {
|
||||
if (pSlowBuf) {
|
||||
|
|
@ -1032,6 +1197,26 @@ static void *taosAsyncOutputLog(void *param) {
|
|||
if (pSlowBuf) taosWriteSlowLog(pSlowBuf);
|
||||
break;
|
||||
}
|
||||
|
||||
// process the log rotation every LOG_ROTATE_INTERVAL
|
||||
int64_t curSec = taosGetTimestampMs() / 1000;
|
||||
if (curSec >= lastCheckSec) {
|
||||
if ((curSec - lastCheckSec) >= LOG_ROTATE_INTERVAL) {
|
||||
TdThread thread;
|
||||
TdThreadAttr attr;
|
||||
(void)taosThreadAttrInit(&attr);
|
||||
(void)taosThreadAttrSetDetachState(&attr, PTHREAD_CREATE_DETACHED);
|
||||
if (taosThreadCreate(&thread, &attr, taosLogRotateFunc, tsLogObj.logHandle) == 0) {
|
||||
uInfo("process log rotation");
|
||||
lastCheckSec = curSec;
|
||||
} else {
|
||||
uWarn("failed to create thread to process log rotation");
|
||||
}
|
||||
(void)taosThreadAttrDestroy(&attr);
|
||||
}
|
||||
} else if (curSec < lastCheckSec) {
|
||||
lastCheckSec = curSec;
|
||||
}
|
||||
}
|
||||
|
||||
return NULL;
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
#define _DEFAULT_SOURCE
|
||||
#include "tutil.h"
|
||||
#include "tlog.h"
|
||||
#include "regex.h"
|
||||
|
||||
void *tmemmem(const char *haystack, int32_t hlen, const char *needle, int32_t nlen) {
|
||||
const char *limit;
|
||||
|
|
@ -520,3 +521,29 @@ int32_t parseCfgReal(const char *str, float *out) {
|
|||
*out = val;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
bool tIsValidFileName(const char *fileName, const char *pattern) {
|
||||
const char *fileNamePattern = "^[a-zA-Z0-9_.-]+$";
|
||||
|
||||
regex_t fileNameReg;
|
||||
|
||||
if (pattern) fileNamePattern = pattern;
|
||||
|
||||
if (regcomp(&fileNameReg, fileNamePattern, REG_EXTENDED) != 0) {
|
||||
fprintf(stderr, "failed to compile file name pattern:%s\n", fileNamePattern);
|
||||
return false;
|
||||
}
|
||||
|
||||
int32_t code = regexec(&fileNameReg, fileName, 0, NULL, 0);
|
||||
regfree(&fileNameReg);
|
||||
if (code != 0) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool tIsValidFilePath(const char *filePath, const char *pattern) {
|
||||
const char *filePathPattern = "^[a-zA-Z0-9:/\\_.-]+$";
|
||||
|
||||
return tIsValidFileName(filePath, pattern ? pattern : filePathPattern);
|
||||
}
|
||||
|
|
@ -354,6 +354,7 @@
|
|||
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/telemetry.py
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/backquote_check.py
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdMonitor.py
|
||||
,,n,system-test,python3 ./test.py -f 0-others/taosdlog.py
|
||||
,,n,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3
|
||||
,,n,system-test,python3 ./test.py -f 0-others/udfTest.py
|
||||
,,n,system-test,python3 ./test.py -f 0-others/udf_create.py
|
||||
|
|
|
|||
|
|
@ -1,66 +1,351 @@
|
|||
import taos
|
||||
import sys
|
||||
import time
|
||||
import os
|
||||
|
||||
from util.log import *
|
||||
from util.sql import *
|
||||
from util.cases import *
|
||||
from util.dnodes import *
|
||||
|
||||
class TDTestCase:
|
||||
|
||||
def init(self, conn, logSql, replicaVar=1):
|
||||
self.replicaVar = int(replicaVar)
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
tdSql.init(conn.cursor())
|
||||
|
||||
def getBuildPath(self):
|
||||
selfPath = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
if ("community" in selfPath):
|
||||
projPath = selfPath[:selfPath.find("community")]
|
||||
else:
|
||||
projPath = selfPath[:selfPath.find("tests")]
|
||||
|
||||
for root, dirs, files in os.walk(projPath):
|
||||
if ("taosd" in files or "taosd.exe" in files):
|
||||
rootRealPath = os.path.dirname(os.path.realpath(root))
|
||||
if ("packaging" not in rootRealPath):
|
||||
buildPath = root[:len(root) - len("/build/bin")]
|
||||
break
|
||||
return buildPath
|
||||
|
||||
def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
|
||||
tdSql.prepare()
|
||||
# time.sleep(2)
|
||||
tdSql.query("create user testpy pass 'testpy'")
|
||||
|
||||
buildPath = self.getBuildPath()
|
||||
if (buildPath == ""):
|
||||
tdLog.exit("taosd not found!")
|
||||
else:
|
||||
tdLog.info("taosd found in %s" % buildPath)
|
||||
logPath = buildPath + "/../sim/dnode1/log"
|
||||
tdLog.info("log path: %s" % logPath)
|
||||
|
||||
tdDnodes.stop(1)
|
||||
time.sleep(2)
|
||||
tdSql.error("select * from information_schema.ins_databases")
|
||||
os.system("rm -rf %s" % logPath)
|
||||
if os.path.exists(logPath) == True:
|
||||
tdLog.exit("log pat still exist!")
|
||||
|
||||
tdDnodes.start(1)
|
||||
time.sleep(2)
|
||||
if os.path.exists(logPath) != True:
|
||||
tdLog.exit("log pat is not generated!")
|
||||
|
||||
tdSql.query("select * from information_schema.ins_databases")
|
||||
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
import concurrent.futures
|
||||
import os
|
||||
import os.path
|
||||
import platform
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from util.log import *
|
||||
from util.sql import *
|
||||
from util.cases import *
|
||||
from util.dnodes import *
|
||||
|
||||
class TDTestCase:
|
||||
|
||||
def init(self, conn, logSql, replicaVar=1):
|
||||
self.replicaVar = int(replicaVar)
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
tdSql.init(conn.cursor())
|
||||
tdSql.prepare()
|
||||
self.buildPath = self.getBuildPath()
|
||||
if (self.buildPath == ""):
|
||||
tdLog.exit("taosd not found!")
|
||||
else:
|
||||
tdLog.info("taosd found in %s" % self.buildPath)
|
||||
self.taosdPath = self.buildPath + "/build/bin/taosd"
|
||||
self.taosPath = self.buildPath + "/build/bin/taos"
|
||||
self.logPath = self.buildPath + "/../sim/dnode1/log"
|
||||
tdLog.info("taosd path: %s" % self.taosdPath)
|
||||
tdLog.info("taos path: %s" % self.taosPath)
|
||||
tdLog.info("log path: %s" % self.logPath)
|
||||
self.commonCfgDict = {}
|
||||
|
||||
def getBuildPath(self):
|
||||
selfPath = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
if ("community" in selfPath):
|
||||
projPath = selfPath[:selfPath.find("community")]
|
||||
else:
|
||||
projPath = selfPath[:selfPath.find("tests")]
|
||||
|
||||
for root, dirs, files in os.walk(projPath):
|
||||
if ("taosd" in files or "taosd.exe" in files):
|
||||
rootRealPath = os.path.dirname(os.path.realpath(root))
|
||||
if ("packaging" not in rootRealPath):
|
||||
buildPath = root[:len(root) - len("/build/bin")]
|
||||
break
|
||||
return buildPath
|
||||
|
||||
def prepareCfg(self, cfgPath, cfgDict):
|
||||
tdLog.info("make dir %s" % cfgPath)
|
||||
os.makedirs(cfgPath, exist_ok=True)
|
||||
with open(cfgPath + "/taos.cfg", "w") as f:
|
||||
for key in self.commonCfgDict:
|
||||
f.write("%s %s\n" % (key, self.commonCfgDict[key]))
|
||||
for key in cfgDict:
|
||||
f.write("%s %s\n" % (key, cfgDict[key]))
|
||||
if not os.path.exists(cfgPath + "/taos.cfg"):
|
||||
tdLog.exit("taos.cfg not found in %s" % cfgPath)
|
||||
else:
|
||||
tdLog.info("taos.cfg found in %s" % cfgPath)
|
||||
|
||||
def closeBin(self, binName):
|
||||
tdLog.info("Closing %s" % binName)
|
||||
if platform.system().lower() == 'windows':
|
||||
psCmd = "for /f %%a in ('wmic process where \"name='%s.exe'\" get processId ^| xargs echo ^| awk ^'{print $2}^' ^&^& echo aa') do @(ps | grep %%a | awk '{print $1}' | xargs)" % binName
|
||||
else:
|
||||
psCmd = "ps -ef | grep -w %s | grep -v grep | awk '{print $2}'" % binName
|
||||
tdLog.info(f"psCmd:{psCmd}")
|
||||
|
||||
try:
|
||||
rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
processID = rem.stdout.decode().strip()
|
||||
tdLog.info(f"processID:{processID}")
|
||||
except Exception as e:
|
||||
tdLog.info(f"closeBin error:{e}")
|
||||
processID = ""
|
||||
onlyKillOnceWindows = 0
|
||||
while(processID):
|
||||
if not platform.system().lower() == 'windows' or (onlyKillOnceWindows == 0 and platform.system().lower() == 'windows'):
|
||||
killCmd = "kill -9 %s > /dev/null 2>&1" % processID
|
||||
if platform.system().lower() == 'windows':
|
||||
killCmd = "kill -9 %s > nul 2>&1" % processID
|
||||
tdLog.info(f"killCmd:{killCmd}")
|
||||
os.system(killCmd)
|
||||
tdLog.info(f"killed {binName} process {processID}")
|
||||
onlyKillOnceWindows = 1
|
||||
time.sleep(1)
|
||||
try:
|
||||
rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
processID = rem.stdout.decode().strip()
|
||||
except Exception as e:
|
||||
tdLog.info(f"closeBin error:{e}")
|
||||
processID = ""
|
||||
|
||||
def closeTaosd(self, signal=9):
|
||||
self.closeBin("taosd")
|
||||
|
||||
def closeTaos(self, signal=9):
|
||||
self.closeBin("taos")
|
||||
|
||||
def openBin(self, binPath, waitSec=5):
|
||||
tdLog.info(f"Opening {binPath}")
|
||||
try:
|
||||
process = subprocess.Popen(binPath, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
time.sleep(waitSec)
|
||||
if process.poll() is None:
|
||||
tdLog.info(f"{binPath} started successfully")
|
||||
else:
|
||||
error = process.stderr.read().decode(encoding="utf-8").strip()
|
||||
raise Exception(f"Failed to start {binPath}: {error}")
|
||||
except Exception as e:
|
||||
raise Exception(f"Failed to start {binPath}: %s" % repr(e))
|
||||
|
||||
def openTaosd(self, args="", waitSec=8):
|
||||
self.openBin(f'{self.taosdPath} {args}', waitSec)
|
||||
|
||||
def openTaos(self, args="", waitSec=5):
|
||||
self.openBin(f'{self.taosPath} {args}', waitSec)
|
||||
|
||||
|
||||
def prepare_logoutput(self, desc, port, logOutput, skipOpenBin=False):
|
||||
tdLog.info("Preparing %s, port:%s" % (desc, port))
|
||||
dnodePath = self.buildPath + "/../sim/dnode%s" % port
|
||||
tdLog.info("remove dnodePath:%s" % dnodePath)
|
||||
try:
|
||||
if os.path.exists(dnodePath):
|
||||
shutil.rmtree(dnodePath)
|
||||
except Exception as e:
|
||||
raise Exception(f"Failed to remove directory {dnodePath}: {e}")
|
||||
try:
|
||||
self.prepareCfg(dnodePath, {"serverPort": port,
|
||||
"dataDir": dnodePath + os.sep + "data",
|
||||
"logDir": dnodePath + os.sep + "log"})
|
||||
except Exception as e:
|
||||
raise Exception(f"Failed to prepare configuration for {dnodePath}: {e}")
|
||||
try:
|
||||
self.openTaosd(f"-c {dnodePath} {logOutput}")
|
||||
self.openTaos(f"-c {dnodePath} {logOutput}")
|
||||
except Exception as e:
|
||||
if(skipOpenBin):
|
||||
tdLog.info(f"Failed to prepare taosd and taos with log output {logOutput}: {e}")
|
||||
else:
|
||||
raise Exception(f"Failed to prepare taosd and taos with log output {logOutput}: {e}")
|
||||
|
||||
|
||||
def prepare_stdout(self):
|
||||
list = self.prepare_list[0]
|
||||
self.prepare_logoutput(list[0], list[1], "-o " + list[0])
|
||||
|
||||
def check_stdout(self):
|
||||
tdLog.info("Running check stdout")
|
||||
dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[0][1]
|
||||
tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0"))
|
||||
tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0"))
|
||||
|
||||
def prepare_stderr(self):
|
||||
list = self.prepare_list[1]
|
||||
self.prepare_logoutput(list[0], list[1], "--log-output " + list[0])
|
||||
|
||||
def check_stderr(self):
|
||||
tdLog.info("Running check stderr")
|
||||
dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[1][1]
|
||||
tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0"))
|
||||
tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0"))
|
||||
|
||||
def prepare_dev_null(self):
|
||||
list = self.prepare_list[2]
|
||||
# self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0])
|
||||
|
||||
def check_dev_null(self):
|
||||
tdLog.info("Running check /dev/null")
|
||||
# dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[2][1]
|
||||
# tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0"))
|
||||
# tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0"))
|
||||
|
||||
def prepare_fullpath(self):
|
||||
list = self.prepare_list[3]
|
||||
dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[3][1]
|
||||
self.prepare_logoutput(list[0], list[1], "-o " + dnodePath + "/log0/" )
|
||||
|
||||
def check_fullpath(self):
|
||||
tdLog.info("Running check fullpath")
|
||||
logPath = self.buildPath + "/../sim/dnode%s/log0/" % self.prepare_list[3][1]
|
||||
tdSql.checkEqual(True, os.path.exists(f"{logPath}taosdlog.0"))
|
||||
tdSql.checkEqual(True, os.path.exists(f"{logPath}taoslog0.0"))
|
||||
|
||||
def prepare_fullname(self):
|
||||
list = self.prepare_list[4]
|
||||
dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[4][1]
|
||||
self.prepare_logoutput(list[0], list[1], "--log-output " + dnodePath + "/log0/" + list[0])
|
||||
dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[4][1]
|
||||
|
||||
def check_fullname(self):
|
||||
tdLog.info("Running check fullname")
|
||||
logPath = self.buildPath + "/../sim/dnode%s/log0/" % self.prepare_list[4][1]
|
||||
tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[4][0] + ".0"))
|
||||
tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[4][0] + "0.0"))
|
||||
|
||||
def prepare_relativepath(self):
|
||||
list = self.prepare_list[5]
|
||||
self.prepare_logoutput(list[0], list[1], "--log-output=" + "log0/")
|
||||
|
||||
def check_relativepath(self):
|
||||
tdLog.info("Running check relativepath")
|
||||
logPath = self.buildPath + "/../sim/dnode%s/log/log0/" % self.prepare_list[5][1]
|
||||
tdSql.checkEqual(True, os.path.exists(logPath + "taosdlog.0"))
|
||||
tdSql.checkEqual(True, os.path.exists(logPath + "taoslog0.0"))
|
||||
|
||||
def prepare_relativename(self):
|
||||
list = self.prepare_list[6]
|
||||
self.prepare_logoutput(list[0], list[1], "-o " + "log0/" + list[0])
|
||||
def check_relativename(self):
|
||||
tdLog.info("Running check relativename")
|
||||
logPath = self.buildPath + "/../sim/dnode%s/log/log0/" % self.prepare_list[6][1]
|
||||
tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[6][0] + ".0"))
|
||||
tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[6][0] + "0.0"))
|
||||
|
||||
def prepare_filename(self):
|
||||
list = self.prepare_list[7]
|
||||
self.prepare_logoutput(list[0], list[1], "--log-output " + list[0])
|
||||
def check_filename(self):
|
||||
tdLog.info("Running check filename")
|
||||
logPath = self.buildPath + "/../sim/dnode%s/log/" % self.prepare_list[7][1]
|
||||
tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[7][0] + ".0"))
|
||||
tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[7][0] + "0.0"))
|
||||
|
||||
def prepare_empty(self):
|
||||
list = self.prepare_list[8]
|
||||
self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0], True)
|
||||
def check_empty(self):
|
||||
tdLog.info("Running check empty")
|
||||
logPath = self.buildPath + "/../sim/dnode%s/log" % self.prepare_list[8][1]
|
||||
tdSql.checkEqual(False, os.path.exists(f"{logPath}/taosdlog.0"))
|
||||
tdSql.checkEqual(False, os.path.exists(f"{logPath}/taoslog.0"))
|
||||
|
||||
def prepare_illegal(self):
|
||||
list = self.prepare_list[9]
|
||||
self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0], True)
|
||||
def check_illegal(self):
|
||||
tdLog.info("Running check empty")
|
||||
logPath = self.buildPath + "/../sim/dnode%s/log" % self.prepare_list[9][1]
|
||||
tdSql.checkEqual(False, os.path.exists(f"{logPath}/taosdlog.0"))
|
||||
tdSql.checkEqual(False, os.path.exists(f"{logPath}/taoslog.0"))
|
||||
|
||||
def prepareCheckResources(self):
|
||||
self.prepare_list = [["stdout", "10030"], ["stderr", "10031"], ["/dev/null", "10032"], ["fullpath", "10033"],
|
||||
["fullname", "10034"], ["relativepath", "10035"], ["relativename", "10036"], ["filename", "10037"],
|
||||
["empty", "10038"], ["illeg?al", "10039"]]
|
||||
self.check_functions = {
|
||||
self.prepare_stdout: self.check_stdout,
|
||||
self.prepare_stderr: self.check_stderr,
|
||||
self.prepare_dev_null: self.check_dev_null,
|
||||
self.prepare_fullpath: self.check_fullpath,
|
||||
self.prepare_fullname: self.check_fullname,
|
||||
self.prepare_relativepath: self.check_relativepath,
|
||||
self.prepare_relativename: self.check_relativename,
|
||||
self.prepare_filename: self.check_filename,
|
||||
self.prepare_empty: self.check_empty,
|
||||
self.prepare_illegal: self.check_illegal,
|
||||
}
|
||||
|
||||
def checkLogOutput(self):
|
||||
self.closeTaosd()
|
||||
self.closeTaos()
|
||||
self.prepareCheckResources()
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
prepare_futures = [executor.submit(prepare_func) for prepare_func, _ in self.check_functions.items()]
|
||||
for future in concurrent.futures.as_completed(prepare_futures):
|
||||
try:
|
||||
future.result()
|
||||
except Exception as e:
|
||||
raise Exception(f"Error in prepare function: {e}")
|
||||
|
||||
check_futures = [executor.submit(check_func) for _, check_func in self.check_functions.items()]
|
||||
for future in concurrent.futures.as_completed(check_futures):
|
||||
try:
|
||||
future.result()
|
||||
except Exception as e:
|
||||
raise Exception(f"Error in prepare function: {e}")
|
||||
|
||||
def checkLogRotate(self):
|
||||
tdLog.info("Running check log rotate")
|
||||
dnodePath = self.buildPath + "/../sim/dnode10050"
|
||||
logRotateAfterBoot = 6 # LOG_ROTATE_BOOT
|
||||
self.closeTaosd()
|
||||
self.closeTaos()
|
||||
try:
|
||||
if os.path.exists(dnodePath):
|
||||
shutil.rmtree(dnodePath)
|
||||
self.prepareCfg(dnodePath, {"serverPort": 10050,
|
||||
"dataDir": dnodePath + os.sep + "data",
|
||||
"logDir": dnodePath + os.sep + "log",
|
||||
"logKeepDays": "3" })
|
||||
except Exception as e:
|
||||
raise Exception(f"Failed to prepare configuration for {dnodePath}: {e}")
|
||||
|
||||
nowSec = int(time.time())
|
||||
stubFile99 = f"{dnodePath}/log/taosdlog.99"
|
||||
stubFile101 = f"{dnodePath}/log/taosdlog.101"
|
||||
stubGzFile98 = f"{dnodePath}/log/taosdlog.98.gz"
|
||||
stubGzFile102 = f"{dnodePath}/log/taosdlog.102.gz"
|
||||
stubFileNow = f"{dnodePath}/log/taosdlog.{nowSec}"
|
||||
stubGzFileNow = f"{dnodePath}/log/taosdlog.%d.gz" % (nowSec - 1)
|
||||
stubGzFileKeep = f"{dnodePath}/log/taosdlog.%d.gz" % (nowSec - 86400 * 2)
|
||||
stubGzFileDel = f"{dnodePath}/log/taosdlog.%d.gz" % (nowSec - 86400 * 3)
|
||||
stubFiles = [stubFile99, stubFile101, stubGzFile98, stubGzFile102, stubFileNow, stubGzFileNow, stubGzFileKeep, stubGzFileDel]
|
||||
|
||||
try:
|
||||
os.makedirs(f"{dnodePath}/log", exist_ok=True)
|
||||
for stubFile in stubFiles:
|
||||
with open(stubFile, "w") as f:
|
||||
f.write("test log rotate")
|
||||
except Exception as e:
|
||||
raise Exception(f"Failed to prepare log files for {dnodePath}: {e}")
|
||||
|
||||
tdSql.checkEqual(True, os.path.exists(stubFile101))
|
||||
tdSql.checkEqual(True, os.path.exists(stubGzFile102))
|
||||
tdSql.checkEqual(True, os.path.exists(stubFileNow))
|
||||
tdSql.checkEqual(True, os.path.exists(stubGzFileDel))
|
||||
|
||||
self.openTaosd(f"-c {dnodePath}")
|
||||
self.openTaos(f"-c {dnodePath}")
|
||||
|
||||
tdLog.info("wait %d seconds for log rotate" % (logRotateAfterBoot + 2))
|
||||
time.sleep(logRotateAfterBoot + 2)
|
||||
|
||||
tdSql.checkEqual(True, os.path.exists(stubFile99))
|
||||
tdSql.checkEqual(False, os.path.exists(stubFile101))
|
||||
tdSql.checkEqual(False, os.path.exists(f'{stubFile101}.gz'))
|
||||
tdSql.checkEqual(True, os.path.exists(stubGzFile98))
|
||||
tdSql.checkEqual(True, os.path.exists(f'{stubFileNow}.gz'))
|
||||
tdSql.checkEqual(True, os.path.exists(stubGzFileNow))
|
||||
tdSql.checkEqual(True, os.path.exists(stubGzFileKeep))
|
||||
tdSql.checkEqual(False, os.path.exists(stubGzFile102))
|
||||
tdSql.checkEqual(False, os.path.exists(stubGzFileDel))
|
||||
tdSql.checkEqual(True, os.path.exists(f"{dnodePath}/log/taosdlog.0"))
|
||||
tdSql.checkEqual(True, os.path.exists(f"{dnodePath}/log/taoslog0.0"))
|
||||
|
||||
def run(self):
|
||||
self.checkLogOutput()
|
||||
self.checkLogRotate()
|
||||
self.closeTaosd()
|
||||
self.closeTaos()
|
||||
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
|
|
|
|||
|
|
@ -49,7 +49,12 @@
|
|||
#define SHELL_PKT_LEN "Packet length used for net test, default is 1024 bytes."
|
||||
#define SHELL_PKT_NUM "Packet numbers used for net test, default is 100."
|
||||
#define SHELL_BI_MODE "Set BI mode"
|
||||
#define SHELL_VERSION "Print program version."
|
||||
#define SHELL_LOG_OUTPUT \
|
||||
"Specify log output. Options:\n\r\t\t\t stdout, stderr, /dev/null, <directory>, <directory>/<filename>, " \
|
||||
"<filename>\n\r\t\t\t * If OUTPUT contains an absolute directory, logs will be stored in that directory " \
|
||||
"instead of logDir.\n\r\t\t\t * If OUTPUT contains a relative directory, logs will be stored in the directory " \
|
||||
"combined with logDir and the relative directory."
|
||||
#define SHELL_VERSION "Print program version."
|
||||
|
||||
#ifdef WEBSOCKET
|
||||
#define SHELL_DSN "Use dsn to connect to the cloud server or to a remote server which provides WebSocket connection."
|
||||
|
|
@ -74,6 +79,9 @@ void shellPrintHelp() {
|
|||
printf("%s%s%s%s\r\n", indent, "-l,", indent, SHELL_PKT_LEN);
|
||||
printf("%s%s%s%s\r\n", indent, "-n,", indent, SHELL_NET_ROLE);
|
||||
printf("%s%s%s%s\r\n", indent, "-N,", indent, SHELL_PKT_NUM);
|
||||
#if defined(LINUX)
|
||||
printf("%s%s%s%s\r\n", indent, "-o,", indent, SHELL_LOG_OUTPUT);
|
||||
#endif
|
||||
printf("%s%s%s%s\r\n", indent, "-p,", indent, SHELL_PASSWORD);
|
||||
printf("%s%s%s%s\r\n", indent, "-P,", indent, SHELL_PORT);
|
||||
printf("%s%s%s%s\r\n", indent, "-r,", indent, SHELL_RAW_TIME);
|
||||
|
|
@ -134,6 +142,9 @@ static struct argp_option shellOptions[] = {
|
|||
#endif
|
||||
{"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM},
|
||||
{"bimode", 'B', 0, 0, SHELL_BI_MODE},
|
||||
#if defined(LINUX)
|
||||
{"log-output", 'o', "OUTPUT", 0, SHELL_LOG_OUTPUT},
|
||||
#endif
|
||||
{0},
|
||||
};
|
||||
|
||||
|
|
@ -222,6 +233,23 @@ static int32_t shellParseSingleOpt(int32_t key, char *arg) {
|
|||
case 'N':
|
||||
pArgs->pktNum = atoi(arg);
|
||||
break;
|
||||
#if defined(LINUX)
|
||||
case 'o':
|
||||
if (strlen(arg) >= PATH_MAX) {
|
||||
printf("failed to set log output since length overflow, max length is %d\n", PATH_MAX);
|
||||
return TSDB_CODE_INVALID_CFG;
|
||||
}
|
||||
tsLogOutput = taosMemoryMalloc(PATH_MAX);
|
||||
if (!tsLogOutput) {
|
||||
printf("failed to set log output: '%s' since %s\n", arg, tstrerror(terrno));
|
||||
return terrno;
|
||||
}
|
||||
if (taosExpandDir(arg, tsLogOutput, PATH_MAX) != 0) {
|
||||
printf("failed to expand log output: '%s' since %s\n", arg, tstrerror(terrno));
|
||||
return terrno;
|
||||
}
|
||||
break;
|
||||
#endif
|
||||
#ifdef WEBSOCKET
|
||||
case 'R':
|
||||
pArgs->restful = true;
|
||||
|
|
|
|||
Loading…
Reference in New Issue