Merge branch '3.0' of https://github.com/taosdata/TDengine into enh/TD-29367-3.0
This commit is contained in:
Hongze Cheng
commit
23fb0fcc24
|
|
@ -29,6 +29,7 @@ This document details the server error codes that may be encountered when using
|
|||
| 0x8000002A | RPC state already dropped | 1. Engine error, can be ignored, this error code will not be returned to the user side | If returned to the user side, the engine side needs to investigate the issue |
|
||||
| 0x8000002B | RPC msg exceed limit | 1. Single RPC message exceeds the limit, this error code will not be returned to the user side | If returned to the user side, the engine side needs to investigate the issue |
|
||||
|
||||
|
||||
## common
|
||||
|
||||
| Error Code | Error Description | Possible Error Scenarios or Reasons | Recommended User Actions |
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ typedef struct {
|
|||
|
||||
char file_path[TSDB_FILENAME_LEN]; // local file path
|
||||
int64_t file_size; // local file size, for upload
|
||||
int32_t file_last_modified; // local file last modified time, for upload
|
||||
int64_t file_last_modified; // local file last modified time, for upload
|
||||
char file_md5[64]; // md5 of the local file content, for upload, reserved
|
||||
|
||||
char object_name[128]; // object name
|
||||
|
|
@ -67,9 +67,9 @@ int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint);
|
|||
int32_t cos_cp_dump(SCheckpoint* checkpoint);
|
||||
void cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPart* parts, int64_t* consume_bytes);
|
||||
void cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag, uint64_t crc64);
|
||||
void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int32_t mtime,
|
||||
void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int64_t mtime,
|
||||
char const* upload_id, int64_t part_size);
|
||||
bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int32_t mtime);
|
||||
bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int64_t mtime);
|
||||
|
||||
void cos_cp_build_download(SCheckpoint* checkpoint, char const* filepath, char const* object_name, int64_t object_size,
|
||||
char const* object_lmtime, char const* object_etag, int64_t part_size);
|
||||
|
|
|
|||
|
|
@ -82,9 +82,9 @@ int32_t taosUnLockFile(TdFilePtr pFile);
|
|||
|
||||
int32_t taosUmaskFile(int32_t maskVal);
|
||||
|
||||
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *atime);
|
||||
int32_t taosStatFile(const char *path, int64_t *size, int64_t *mtime, int64_t *atime);
|
||||
int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno);
|
||||
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime);
|
||||
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int64_t *mtime);
|
||||
bool taosCheckExistFile(const char *pathname);
|
||||
|
||||
int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence);
|
||||
|
|
|
|||
|
|
@ -196,21 +196,21 @@ int32_t taosGetErrSize();
|
|||
#define TSDB_CODE_TSC_INVALID_JSON TAOS_DEF_ERROR_CODE(0, 0x0221)
|
||||
#define TSDB_CODE_TSC_INVALID_JSON_TYPE TAOS_DEF_ERROR_CODE(0, 0x0222)
|
||||
#define TSDB_CODE_TSC_VALUE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0224)
|
||||
#define TSDB_CODE_TSC_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0X0229)
|
||||
#define TSDB_CODE_TSC_STMT_API_ERROR TAOS_DEF_ERROR_CODE(0, 0X022A)
|
||||
#define TSDB_CODE_TSC_STMT_TBNAME_ERROR TAOS_DEF_ERROR_CODE(0, 0X022B)
|
||||
#define TSDB_CODE_TSC_STMT_CLAUSE_ERROR TAOS_DEF_ERROR_CODE(0, 0X022C)
|
||||
#define TSDB_CODE_TSC_QUERY_KILLED TAOS_DEF_ERROR_CODE(0, 0X022D)
|
||||
#define TSDB_CODE_TSC_NO_EXEC_NODE TAOS_DEF_ERROR_CODE(0, 0X022E)
|
||||
#define TSDB_CODE_TSC_NOT_STABLE_ERROR TAOS_DEF_ERROR_CODE(0, 0X022F)
|
||||
#define TSDB_CODE_TSC_STMT_CACHE_ERROR TAOS_DEF_ERROR_CODE(0, 0X0230)
|
||||
#define TSDB_CODE_TSC_ENCODE_PARAM_ERROR TAOS_DEF_ERROR_CODE(0, 0X0231)
|
||||
#define TSDB_CODE_TSC_ENCODE_PARAM_NULL TAOS_DEF_ERROR_CODE(0, 0X0232)
|
||||
#define TSDB_CODE_TSC_COMPRESS_PARAM_ERROR TAOS_DEF_ERROR_CODE(0, 0X0233)
|
||||
#define TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR TAOS_DEF_ERROR_CODE(0, 0X0234)
|
||||
#define TSDB_CODE_TSC_FAIL_GENERATE_JSON TAOS_DEF_ERROR_CODE(0, 0X0235)
|
||||
#define TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR TAOS_DEF_ERROR_CODE(0, 0X0236)
|
||||
#define TSDB_CODE_TSC_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0X02FF)
|
||||
#define TSDB_CODE_TSC_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x0229)
|
||||
#define TSDB_CODE_TSC_STMT_API_ERROR TAOS_DEF_ERROR_CODE(0, 0x022A)
|
||||
#define TSDB_CODE_TSC_STMT_TBNAME_ERROR TAOS_DEF_ERROR_CODE(0, 0x022B)
|
||||
#define TSDB_CODE_TSC_STMT_CLAUSE_ERROR TAOS_DEF_ERROR_CODE(0, 0x022C)
|
||||
#define TSDB_CODE_TSC_QUERY_KILLED TAOS_DEF_ERROR_CODE(0, 0x022D)
|
||||
#define TSDB_CODE_TSC_NO_EXEC_NODE TAOS_DEF_ERROR_CODE(0, 0x022E)
|
||||
#define TSDB_CODE_TSC_NOT_STABLE_ERROR TAOS_DEF_ERROR_CODE(0, 0x022F)
|
||||
#define TSDB_CODE_TSC_STMT_CACHE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0230)
|
||||
#define TSDB_CODE_TSC_ENCODE_PARAM_ERROR TAOS_DEF_ERROR_CODE(0, 0x0231)
|
||||
#define TSDB_CODE_TSC_ENCODE_PARAM_NULL TAOS_DEF_ERROR_CODE(0, 0x0232)
|
||||
#define TSDB_CODE_TSC_COMPRESS_PARAM_ERROR TAOS_DEF_ERROR_CODE(0, 0x0233)
|
||||
#define TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR TAOS_DEF_ERROR_CODE(0, 0x0234)
|
||||
#define TSDB_CODE_TSC_FAIL_GENERATE_JSON TAOS_DEF_ERROR_CODE(0, 0x0235)
|
||||
#define TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR TAOS_DEF_ERROR_CODE(0, 0x0236)
|
||||
#define TSDB_CODE_TSC_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x02FF)
|
||||
|
||||
// mnode-common
|
||||
#define TSDB_CODE_MND_REQ_REJECTED TAOS_DEF_ERROR_CODE(0, 0x0300)
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ extern bool tsAsyncLog;
|
|||
extern bool tsAssert;
|
||||
extern int32_t tsNumOfLogLines;
|
||||
extern int32_t tsLogKeepDays;
|
||||
extern char *tsLogOutput;
|
||||
extern LogFp tsLogFp;
|
||||
extern int64_t tsNumOfErrorLogs;
|
||||
extern int64_t tsNumOfInfoLogs;
|
||||
|
|
@ -71,7 +72,7 @@ extern int32_t sndDebugFlag;
|
|||
extern int32_t simDebugFlag;
|
||||
|
||||
extern int32_t tqClientDebugFlag;
|
||||
|
||||
int32_t taosInitLogOutput(const char **ppLogName);
|
||||
int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc);
|
||||
void taosCloseLog();
|
||||
void taosResetLog();
|
||||
|
|
|
|||
|
|
@ -56,6 +56,8 @@ void taosIpPort2String(uint32_t ip, uint16_t port, char *str);
|
|||
void *tmemmem(const char *haystack, int hlen, const char *needle, int nlen);
|
||||
|
||||
int32_t parseCfgReal(const char *str, float *out);
|
||||
bool tIsValidFileName(const char *fileName, const char *pattern);
|
||||
bool tIsValidFilePath(const char *filePath, const char *pattern);
|
||||
|
||||
static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) {
|
||||
T_MD5_CTX context;
|
||||
|
|
|
|||
|
|
@ -41,6 +41,10 @@
|
|||
#include "cus_name.h"
|
||||
#endif
|
||||
|
||||
#ifndef CUS_PROMPT
|
||||
#define CUS_PROMPT "tao"
|
||||
#endif
|
||||
|
||||
#define TSC_VAR_NOT_RELEASE 1
|
||||
#define TSC_VAR_RELEASED 0
|
||||
|
||||
|
|
@ -954,14 +958,10 @@ void taos_init_imp(void) {
|
|||
taosHashSetFreeFp(appInfo.pInstMap, destroyAppInst);
|
||||
deltaToUtcInitOnce();
|
||||
|
||||
char logDirName[64] = {0};
|
||||
#ifdef CUS_PROMPT
|
||||
snprintf(logDirName, 64, "%slog", CUS_PROMPT);
|
||||
#else
|
||||
(void)snprintf(logDirName, 64, "taoslog");
|
||||
#endif
|
||||
if (taosCreateLog(logDirName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) {
|
||||
(void)printf(" WARING: Create %s failed:%s. configDir=%s\n", logDirName, strerror(errno), configDir);
|
||||
const char *logName = CUS_PROMPT "slog";
|
||||
ENV_ERR_RET(taosInitLogOutput(&logName), "failed to init log output");
|
||||
if (taosCreateLog(logName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) {
|
||||
(void)printf(" WARING: Create %s failed:%s. configDir=%s\n", logName, strerror(errno), configDir);
|
||||
tscInitRes = -1;
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -775,7 +775,7 @@ _exit:
|
|||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const char *file, int32_t lmtime,
|
||||
static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const char *file, int64_t lmtime,
|
||||
char const *object_name, int64_t contentLength, S3PutProperties *put_prop,
|
||||
put_object_callback_data *data) {
|
||||
int32_t code = 0, lino = 0;
|
||||
|
|
@ -963,7 +963,7 @@ _exit:
|
|||
|
||||
int32_t s3PutObjectFromFile2ByEp(const char *file, const char *object_name, int8_t withcp, int8_t epIndex) {
|
||||
int32_t code = 0;
|
||||
int32_t lmtime = 0;
|
||||
int64_t lmtime = 0;
|
||||
const char *filename = 0;
|
||||
uint64_t contentLength = 0;
|
||||
const char *cacheControl = 0, *contentType = 0, *md5 = 0;
|
||||
|
|
@ -1040,7 +1040,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t w
|
|||
static int32_t s3PutObjectFromFileOffsetByEp(const char *file, const char *object_name, int64_t offset, int64_t size,
|
||||
int8_t epIndex) {
|
||||
int32_t code = 0;
|
||||
int32_t lmtime = 0;
|
||||
int64_t lmtime = 0;
|
||||
const char *filename = 0;
|
||||
uint64_t contentLength = 0;
|
||||
const char *cacheControl = 0, *contentType = 0, *md5 = 0;
|
||||
|
|
@ -1847,7 +1847,7 @@ _exit:
|
|||
|
||||
typedef struct {
|
||||
int64_t size;
|
||||
int32_t atime;
|
||||
int64_t atime;
|
||||
char name[TSDB_FILENAME_LEN];
|
||||
} SEvictFile;
|
||||
|
||||
|
|
|
|||
|
|
@ -350,7 +350,7 @@ void cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag
|
|||
checkpoint->parts[part_index].crc64 = crc64;
|
||||
}
|
||||
|
||||
void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int32_t mtime,
|
||||
void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int64_t mtime,
|
||||
char const* upload_id, int64_t part_size) {
|
||||
int i = 0;
|
||||
|
||||
|
|
@ -375,7 +375,7 @@ void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t
|
|||
|
||||
static bool cos_cp_verify_md5(SCheckpoint* cp) { return true; }
|
||||
|
||||
bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int32_t mtime) {
|
||||
bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int64_t mtime) {
|
||||
if (cos_cp_verify_md5(checkpoint) && checkpoint->file_size == size && checkpoint->file_last_modified == mtime) {
|
||||
return true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
#include "tglobal.h"
|
||||
#include "defines.h"
|
||||
#include "os.h"
|
||||
#include "osString.h"
|
||||
#include "tconfig.h"
|
||||
#include "tgrant.h"
|
||||
#include "tlog.h"
|
||||
|
|
@ -1010,12 +1011,43 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
|
|||
TAOS_RETURN(TSDB_CODE_SUCCESS);
|
||||
}
|
||||
|
||||
static int32_t taosSetLogOutput(SConfig *pCfg) {
|
||||
if (tsLogOutput) {
|
||||
char *pLog = tsLogOutput;
|
||||
char *pEnd = NULL;
|
||||
if (strcasecmp(pLog, "stdout") && strcasecmp(pLog, "stderr") && strcasecmp(pLog, "/dev/null")) {
|
||||
if ((pEnd = strrchr(pLog, '/')) || (pEnd = strrchr(pLog, '\\'))) {
|
||||
int32_t pathLen = POINTER_DISTANCE(pEnd, pLog) + 1;
|
||||
if (*pLog == '/' || *pLog == '\\') {
|
||||
if (pathLen <= 0 || pathLen > PATH_MAX) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
|
||||
tstrncpy(tsLogDir, pLog, pathLen);
|
||||
} else {
|
||||
int32_t len = strlen(tsLogDir);
|
||||
if (len < 0 || len >= (PATH_MAX - 1)) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
|
||||
if (len == 0 || (tsLogDir[len - 1] != '/' && tsLogDir[len - 1] != '\\')) {
|
||||
tsLogDir[len++] = TD_DIRSEP_CHAR;
|
||||
}
|
||||
int32_t remain = PATH_MAX - len - 1;
|
||||
if (remain < pathLen) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
|
||||
tstrncpy(tsLogDir + len, pLog, pathLen);
|
||||
}
|
||||
TAOS_CHECK_RETURN(cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true));
|
||||
}
|
||||
} else {
|
||||
tstrncpy(tsLogDir, pLog, PATH_MAX);
|
||||
TAOS_CHECK_RETURN(cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true));
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t taosSetClientLogCfg(SConfig *pCfg) {
|
||||
SConfigItem *pItem = NULL;
|
||||
|
||||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "logDir");
|
||||
tstrncpy(tsLogDir, pItem->str, PATH_MAX);
|
||||
TAOS_CHECK_RETURN(taosExpandDir(tsLogDir, tsLogDir, PATH_MAX));
|
||||
TAOS_CHECK_RETURN(taosSetLogOutput(pCfg));
|
||||
|
||||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "minimalLogDirGB");
|
||||
tsLogSpace.reserved = (int64_t)(((double)pItem->fval) * 1024 * 1024 * 1024);
|
||||
|
|
@ -1860,8 +1892,10 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
|
|||
}
|
||||
|
||||
if (tsc) {
|
||||
TAOS_CHECK_GOTO(taosSetClientLogCfg(tsCfg), &lino, _exit);
|
||||
TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit);
|
||||
} else {
|
||||
TAOS_CHECK_GOTO(taosSetClientLogCfg(tsCfg), &lino, _exit);
|
||||
TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit);
|
||||
TAOS_CHECK_GOTO(taosUpdateServerCfg(tsCfg), &lino, _exit);
|
||||
TAOS_CHECK_GOTO(taosSetServerCfg(tsCfg), &lino, _exit);
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@
|
|||
#define DM_ENV_CMD "The env cmd variable string to use when configuring the server, such as: -e 'TAOS_FQDN=td1'."
|
||||
#define DM_ENV_FILE "The env variable file path to use when configuring the server, default is './.env', .env text can be 'TAOS_FQDN=td1'."
|
||||
#define DM_MACHINE_CODE "Get machine code."
|
||||
#define DM_LOG_OUTPUT "Specify log output. Options:\n\r\t\t\t stdout, stderr, /dev/null, <directory>, <directory>/<filename>, <filename>\n\r\t\t\t * If OUTPUT contains an absolute directory, logs will be stored in that directory instead of logDir.\n\r\t\t\t * If OUTPUT contains a relative directory, logs will be stored in the directory combined with logDir and the relative directory."
|
||||
#define DM_VERSION "Print program version."
|
||||
#define DM_EMAIL "<support@taosdata.com>"
|
||||
#define DM_MEM_DBG "Enable memory debug"
|
||||
|
|
@ -239,6 +240,32 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
|
|||
}
|
||||
} else if (strcmp(argv[i], "-k") == 0) {
|
||||
global.generateGrant = true;
|
||||
#if defined(LINUX)
|
||||
} else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--log-output") == 0 ||
|
||||
strncmp(argv[i], "--log-output=", 13) == 0) {
|
||||
if ((i < argc - 1) || ((i == argc - 1) && strncmp(argv[i], "--log-output=", 13) == 0)) {
|
||||
int32_t klen = strlen(argv[i]);
|
||||
int32_t vlen = klen < 13 ? strlen(argv[++i]) : klen - 13;
|
||||
const char *val = argv[i];
|
||||
if (klen >= 13) val += 13;
|
||||
if (vlen <= 0 || vlen >= PATH_MAX) {
|
||||
printf("failed to set log output since invalid vlen:%d, valid range: [1, %d)\n", vlen, PATH_MAX);
|
||||
return TSDB_CODE_INVALID_CFG;
|
||||
}
|
||||
tsLogOutput = taosMemoryMalloc(PATH_MAX);
|
||||
if (!tsLogOutput) {
|
||||
printf("failed to set log output: '%s' since %s\n", val, tstrerror(terrno));
|
||||
return terrno;
|
||||
}
|
||||
if (taosExpandDir(val, tsLogOutput, PATH_MAX) != 0) {
|
||||
printf("failed to expand log output: '%s' since %s\n", val, tstrerror(terrno));
|
||||
return terrno;
|
||||
}
|
||||
} else {
|
||||
printf("'%s' requires a parameter\n", argv[i]);
|
||||
return TSDB_CODE_INVALID_CFG;
|
||||
}
|
||||
#endif
|
||||
} else if (strcmp(argv[i], "-y") == 0) {
|
||||
global.generateCode = true;
|
||||
if (i < argc - 1) {
|
||||
|
|
@ -316,6 +343,9 @@ static void dmPrintHelp() {
|
|||
printf("%s%s%s%s\n", indent, "-e,", indent, DM_ENV_CMD);
|
||||
printf("%s%s%s%s\n", indent, "-E,", indent, DM_ENV_FILE);
|
||||
printf("%s%s%s%s\n", indent, "-k,", indent, DM_MACHINE_CODE);
|
||||
#if defined(LINUX)
|
||||
printf("%s%s%s%s\n", indent, "-o, --log-output=OUTPUT", indent, DM_LOG_OUTPUT);
|
||||
#endif
|
||||
printf("%s%s%s%s\n", indent, "-y,", indent, DM_SET_ENCRYPTKEY);
|
||||
printf("%s%s%s%s\n", indent, "-dm,", indent, DM_MEM_DBG);
|
||||
printf("%s%s%s%s\n", indent, "-V,", indent, DM_VERSION);
|
||||
|
|
@ -340,8 +370,11 @@ static int32_t dmCheckS3() {
|
|||
}
|
||||
|
||||
static int32_t dmInitLog() {
|
||||
return taosCreateLog(CUS_PROMPT "dlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs,
|
||||
0);
|
||||
const char *logName = CUS_PROMPT "dlog";
|
||||
|
||||
TAOS_CHECK_RETURN(taosInitLogOutput(&logName));
|
||||
|
||||
return taosCreateLog(logName, 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0);
|
||||
}
|
||||
|
||||
static void taosCleanupArgs() {
|
||||
|
|
|
|||
|
|
@ -660,7 +660,7 @@ static int32_t tsdbDoS3Migrate(SRTNer *rtner) {
|
|||
int32_t lcn = fobj->f->lcn;
|
||||
|
||||
if (/*lcn < 1 && */ taosCheckExistFile(fobj->fname)) {
|
||||
int32_t mtime = 0;
|
||||
int64_t mtime = 0;
|
||||
int64_t size = 0;
|
||||
int32_t r = taosStatFile(fobj->fname, &size, &mtime, NULL);
|
||||
if (size > chunksize && mtime < rtner->now - tsS3UploadDelaySec) {
|
||||
|
|
@ -687,7 +687,7 @@ static int32_t tsdbDoS3Migrate(SRTNer *rtner) {
|
|||
tsdbTFileLastChunkName(rtner->tsdb, fobj->f, fname1);
|
||||
|
||||
if (taosCheckExistFile(fname1)) {
|
||||
int32_t mtime = 0;
|
||||
int64_t mtime = 0;
|
||||
int64_t size = 0;
|
||||
if (taosStatFile(fname1, &size, &mtime, NULL) != 0) {
|
||||
tsdbError("vgId:%d, %s failed at %s:%d ", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, __LINE__);
|
||||
|
|
|
|||
|
|
@ -1564,8 +1564,8 @@ static void udfdPrintVersion() {
|
|||
}
|
||||
|
||||
static int32_t udfdInitLog() {
|
||||
char logName[12] = {0};
|
||||
snprintf(logName, sizeof(logName), "%slog", "udfd");
|
||||
const char *logName = "udfdlog";
|
||||
TAOS_CHECK_RETURN(taosInitLogOutput(&logName));
|
||||
return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, NULL, 0);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1724,9 +1724,8 @@ void syncNodeResetElectTimer(SSyncNode* pSyncNode) {
|
|||
electMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine);
|
||||
}
|
||||
|
||||
// TODO check return value
|
||||
if ((code = syncNodeRestartElectTimer(pSyncNode, electMS)) != 0) {
|
||||
sError("vgId:%d, failed to restart elect timer since %s", pSyncNode->vgId, tstrerror(code));
|
||||
sWarn("vgId:%d, failed to restart elect timer since %s", pSyncNode->vgId, tstrerror(code));
|
||||
return;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -1152,11 +1152,11 @@ int32_t syncLogReplProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncApp
|
|||
int32_t code = 0;
|
||||
if (pMgr->restored) {
|
||||
if ((code = syncLogReplContinue(pMgr, pNode, pMsg)) != 0) {
|
||||
sError("vgId:%d, failed to continue sync log repl since %s", pNode->vgId, tstrerror(code));
|
||||
sWarn("vgId:%d, failed to continue sync log repl since %s", pNode->vgId, tstrerror(code));
|
||||
}
|
||||
} else {
|
||||
if ((code = syncLogReplRecover(pMgr, pNode, pMsg)) != 0) {
|
||||
sError("vgId:%d, failed to recover sync log repl since %s", pNode->vgId, tstrerror(code));
|
||||
sWarn("vgId:%d, failed to recover sync log repl since %s", pNode->vgId, tstrerror(code));
|
||||
}
|
||||
}
|
||||
(void)taosThreadMutexUnlock(&pBuf->mutex);
|
||||
|
|
|
|||
|
|
@ -75,8 +75,10 @@ int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode) {
|
|||
continue;
|
||||
}
|
||||
SSyncLogReplMgr* pMgr = pNode->logReplMgrs[i];
|
||||
if (syncLogReplStart(pMgr, pNode) != 0) {
|
||||
sError("vgId:%d, failed to start log replication to dnode:%d", pNode->vgId, DID(&(pNode->replicasId[i])));
|
||||
int32_t ret = 0;
|
||||
if ((ret = syncLogReplStart(pMgr, pNode)) != 0) {
|
||||
sWarn("vgId:%d, failed to start log replication to dnode:%d since %s", pNode->vgId, DID(&(pNode->replicasId[i])),
|
||||
tstrerror(ret));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -326,7 +326,7 @@ static int32_t walRepairLogFileTs(SWal* pWal, bool* updateMeta) {
|
|||
}
|
||||
|
||||
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
|
||||
int32_t mtime = 0;
|
||||
int64_t mtime = 0;
|
||||
if (taosStatFile(fnameStr, NULL, &mtime, NULL) < 0) {
|
||||
wError("vgId:%d, failed to stat file due to %s, file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
|
||||
|
||||
|
|
|
|||
|
|
@ -303,7 +303,7 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) {
|
|||
if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue;
|
||||
|
||||
char filename[1024];
|
||||
(void)snprintf(filename, sizeof(filename), "%s/%s", dirname, taosGetDirEntryName(de));
|
||||
(void)snprintf(filename, sizeof(filename), "%s%s%s", dirname, TD_DIRSEP, taosGetDirEntryName(de));
|
||||
if (taosDirEntryIsDir(de)) {
|
||||
continue;
|
||||
} else {
|
||||
|
|
@ -349,7 +349,7 @@ int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) {
|
|||
wordfree(&full_path);
|
||||
// FALL THROUGH
|
||||
default:
|
||||
return code;
|
||||
return terrno = TSDB_CODE_INVALID_PARA;
|
||||
}
|
||||
|
||||
if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) {
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ int32_t osDefaultInit() {
|
|||
}
|
||||
tstrncpy(tsDataDir, TD_DATA_DIR_PATH, sizeof(tsDataDir));
|
||||
tstrncpy(tsLogDir, TD_LOG_DIR_PATH, sizeof(tsLogDir));
|
||||
if (strlen(tsTempDir) == 0){
|
||||
if (strlen(tsTempDir) == 0) {
|
||||
tstrncpy(tsTempDir, TD_TMP_DIR_PATH, sizeof(tsTempDir));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -273,7 +273,7 @@ int32_t taosRenameFile(const char *oldName, const char *newName) {
|
|||
#endif
|
||||
}
|
||||
|
||||
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *atime) {
|
||||
int32_t taosStatFile(const char *path, int64_t *size, int64_t *mtime, int64_t *atime) {
|
||||
OS_PARAM_CHECK(path);
|
||||
#ifdef WINDOWS
|
||||
struct _stati64 fileStat;
|
||||
|
|
@ -544,7 +544,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) {
|
|||
return liOffset.QuadPart;
|
||||
}
|
||||
|
||||
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) {
|
||||
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int64_t *mtime) {
|
||||
if (pFile == NULL || pFile->hFile == NULL) {
|
||||
terrno = TSDB_CODE_INVALID_PARA;
|
||||
return terrno;
|
||||
|
|
@ -571,7 +571,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) {
|
|||
ULARGE_INTEGER ull;
|
||||
ull.LowPart = lastWriteTime.dwLowDateTime;
|
||||
ull.HighPart = lastWriteTime.dwHighDateTime;
|
||||
*mtime = (int32_t)((ull.QuadPart - 116444736000000000ULL) / 10000000ULL);
|
||||
*mtime = (int64_t)((ull.QuadPart - 116444736000000000ULL) / 10000000ULL);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -937,7 +937,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) {
|
|||
return ret;
|
||||
}
|
||||
|
||||
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) {
|
||||
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int64_t *mtime) {
|
||||
if (pFile == NULL) {
|
||||
terrno = TSDB_CODE_INVALID_PARA;
|
||||
return terrno;
|
||||
|
|
|
|||
|
|
@ -51,6 +51,13 @@
|
|||
#define LOG_EDITION_FLG ("C")
|
||||
#endif
|
||||
|
||||
typedef enum {
|
||||
LOG_OUTPUT_FILE = 0, // default
|
||||
LOG_OUTPUT_STDOUT = 1, // stdout set by -o option on the command line
|
||||
LOG_OUTPUT_STDERR = 2, // stderr set by -o option on the command line
|
||||
LOG_OUTPUT_NULL = 4, // /dev/null set by -o option on the command line
|
||||
} ELogOutputType;
|
||||
|
||||
typedef struct {
|
||||
char *buffer;
|
||||
int32_t buffStart;
|
||||
|
|
@ -73,6 +80,7 @@ typedef struct {
|
|||
int32_t openInProgress;
|
||||
int64_t lastKeepFileSec;
|
||||
int64_t timestampToday;
|
||||
int8_t outputType; // ELogOutputType
|
||||
pid_t pid;
|
||||
char logName[PATH_MAX];
|
||||
char slowLogName[PATH_MAX];
|
||||
|
|
@ -86,6 +94,7 @@ static int8_t tsLogInited = 0;
|
|||
static SLogObj tsLogObj = {.fileNum = 1, .slowHandle = NULL};
|
||||
static int64_t tsAsyncLogLostLines = 0;
|
||||
static int32_t tsDaylightActive; /* Currently in daylight saving time. */
|
||||
static SRWLatch tsLogRotateLatch = 0;
|
||||
|
||||
bool tsLogEmbedded = 0;
|
||||
bool tsAsyncLog = true;
|
||||
|
|
@ -96,6 +105,7 @@ bool tsAssert = true;
|
|||
#endif
|
||||
int32_t tsNumOfLogLines = 10000000;
|
||||
int32_t tsLogKeepDays = 0;
|
||||
char *tsLogOutput = NULL;
|
||||
LogFp tsLogFp = NULL;
|
||||
int64_t tsNumOfErrorLogs = 0;
|
||||
int64_t tsNumOfInfoLogs = 0;
|
||||
|
|
@ -234,6 +244,58 @@ int32_t taosInitSlowLog() {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t taosInitLogOutput(const char **ppLogName) {
|
||||
const char *pLog = tsLogOutput;
|
||||
const char *pLogName = NULL;
|
||||
if (pLog) {
|
||||
if (!tIsValidFilePath(pLog, NULL)) {
|
||||
fprintf(stderr, "invalid log output destination:%s, contains illegal char\n", pLog);
|
||||
return TSDB_CODE_INVALID_CFG;
|
||||
}
|
||||
if (0 == strcasecmp(pLog, "stdout")) {
|
||||
tsLogObj.outputType = LOG_OUTPUT_STDOUT;
|
||||
if (ppLogName) *ppLogName = pLog;
|
||||
return 0;
|
||||
}
|
||||
if (0 == strcasecmp(pLog, "stderr")) {
|
||||
tsLogObj.outputType = LOG_OUTPUT_STDERR;
|
||||
if (ppLogName) *ppLogName = pLog;
|
||||
return 0;
|
||||
}
|
||||
if (0 == strcasecmp(pLog, "/dev/null")) {
|
||||
tsLogObj.outputType = LOG_OUTPUT_NULL;
|
||||
if (ppLogName) *ppLogName = pLog;
|
||||
return 0;
|
||||
}
|
||||
int32_t len = strlen(pLog);
|
||||
if (len < 1) {
|
||||
fprintf(stderr, "invalid log output destination:%s, should not be empty\n", pLog);
|
||||
return TSDB_CODE_INVALID_CFG;
|
||||
}
|
||||
const char *p = pLog + (len - 1);
|
||||
if (*p == '/' || *p == '\\') {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if ((p = strrchr(pLog, '/')) || (p = strrchr(pLog, '\\'))) {
|
||||
pLogName = p + 1;
|
||||
} else {
|
||||
pLogName = pLog;
|
||||
}
|
||||
if (strcmp(pLogName, ".") == 0 || strcmp(pLogName, "..") == 0) {
|
||||
fprintf(stderr, "invalid log output destination:%s\n", pLog);
|
||||
return TSDB_CODE_INVALID_CFG;
|
||||
}
|
||||
|
||||
if (!tIsValidFileName(pLogName, NULL)) {
|
||||
fprintf(stderr, "invalid log output destination:%s, contains illegal char\n", pLog);
|
||||
return TSDB_CODE_INVALID_CFG;
|
||||
}
|
||||
if (ppLogName) *ppLogName = pLogName;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) {
|
||||
if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0;
|
||||
int32_t code = osUpdate();
|
||||
|
|
@ -241,6 +303,11 @@ int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) {
|
|||
uError("failed to update os info, reason:%s", tstrerror(code));
|
||||
}
|
||||
|
||||
if (tsLogObj.outputType == LOG_OUTPUT_STDOUT || tsLogObj.outputType == LOG_OUTPUT_STDERR ||
|
||||
tsLogObj.outputType == LOG_OUTPUT_NULL) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
TAOS_CHECK_RETURN(taosInitNormalLog(logName, maxFiles));
|
||||
if (tsc) {
|
||||
TAOS_CHECK_RETURN(taosInitSlowLog());
|
||||
|
|
@ -283,6 +350,7 @@ void taosCloseLog() {
|
|||
taosMemoryFreeClear(tsLogObj.logHandle);
|
||||
tsLogObj.logHandle = NULL;
|
||||
}
|
||||
taosMemoryFreeClear(tsLogOutput);
|
||||
}
|
||||
|
||||
static bool taosLockLogFile(TdFilePtr pFile) {
|
||||
|
|
@ -340,10 +408,6 @@ static void taosKeepOldLog(char *oldName) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (tsLogKeepDays > 0) {
|
||||
taosRemoveOldFiles(tsLogDir, tsLogKeepDays);
|
||||
}
|
||||
}
|
||||
typedef struct {
|
||||
TdFilePtr pOldFile;
|
||||
|
|
@ -392,11 +456,16 @@ static OldFileKeeper *taosOpenNewFile() {
|
|||
|
||||
static void *taosThreadToCloseOldFile(void *param) {
|
||||
if (!param) return NULL;
|
||||
taosWLockLatch(&tsLogRotateLatch);
|
||||
OldFileKeeper *oldFileKeeper = (OldFileKeeper *)param;
|
||||
taosSsleep(20);
|
||||
taosCloseLogByFd(oldFileKeeper->pOldFile);
|
||||
taosKeepOldLog(oldFileKeeper->keepName);
|
||||
taosMemoryFree(oldFileKeeper);
|
||||
if (tsLogKeepDays > 0) {
|
||||
taosRemoveOldFiles(tsLogDir, tsLogKeepDays);
|
||||
}
|
||||
taosWUnLockLatch(&tsLogRotateLatch);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
|
@ -532,8 +601,8 @@ static void decideLogFileName(const char *fn, int32_t maxFileNum) {
|
|||
|
||||
static void decideLogFileNameFlag() {
|
||||
char name[PATH_MAX + 50] = "\0";
|
||||
int32_t logstat0_mtime = 0;
|
||||
int32_t logstat1_mtime = 0;
|
||||
int64_t logstat0_mtime = 0;
|
||||
int64_t logstat1_mtime = 0;
|
||||
bool log0Exist = false;
|
||||
bool log1Exist = false;
|
||||
|
||||
|
|
@ -680,10 +749,19 @@ static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *b
|
|||
}
|
||||
}
|
||||
|
||||
if (dflag & DEBUG_SCREEN) {
|
||||
int fd = 0;
|
||||
if (tsLogObj.outputType == LOG_OUTPUT_FILE) {
|
||||
if (dflag & DEBUG_SCREEN) fd = 1;
|
||||
} else if (tsLogObj.outputType == LOG_OUTPUT_STDOUT) {
|
||||
fd = 1;
|
||||
} else if (tsLogObj.outputType == LOG_OUTPUT_STDERR) {
|
||||
fd = 2;
|
||||
}
|
||||
|
||||
if (fd) {
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wunused-result"
|
||||
if (write(1, buffer, (uint32_t)len) < 0) {
|
||||
if (write(fd, buffer, (uint32_t)len) < 0) {
|
||||
TAOS_UNUSED(printf("failed to write log to screen, reason:%s\n", strerror(errno)));
|
||||
}
|
||||
#pragma GCC diagnostic pop
|
||||
|
|
@ -999,6 +1077,92 @@ static void taosWriteLog(SLogBuff *pLogBuf) {
|
|||
pLogBuf->writeInterval = 0;
|
||||
}
|
||||
|
||||
#define LOG_ROTATE_INTERVAL 3600
|
||||
#if !defined(TD_ENTERPRISE) || defined(ASSERT_NOT_CORE)
|
||||
#define LOG_INACTIVE_TIME 7200
|
||||
#define LOG_ROTATE_BOOT 900
|
||||
#else
|
||||
#define LOG_INACTIVE_TIME 5
|
||||
#define LOG_ROTATE_BOOT (LOG_INACTIVE_TIME + 1)
|
||||
#endif
|
||||
|
||||
static void *taosLogRotateFunc(void *param) {
|
||||
setThreadName("logRotate");
|
||||
int32_t code = 0;
|
||||
taosWLockLatch(&tsLogRotateLatch);
|
||||
// compress or remove the old log files
|
||||
TdDirPtr pDir = taosOpenDir(tsLogDir);
|
||||
if (!pDir) goto _exit;
|
||||
TdDirEntryPtr de = NULL;
|
||||
while ((de = taosReadDir(pDir))) {
|
||||
if (taosDirEntryIsDir(de)) {
|
||||
continue;
|
||||
}
|
||||
char *fname = taosGetDirEntryName(de);
|
||||
if (!fname) {
|
||||
continue;
|
||||
}
|
||||
|
||||
char *pSec = strrchr(fname, '.');
|
||||
if (!pSec) {
|
||||
continue;
|
||||
}
|
||||
char *pIter = pSec;
|
||||
bool isSec = true;
|
||||
while (*(++pIter)) {
|
||||
if (!isdigit(*pIter)) {
|
||||
isSec = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!isSec) {
|
||||
continue;
|
||||
}
|
||||
|
||||
int64_t fileSec = 0;
|
||||
if ((code = taosStr2int64(pSec + 1, &fileSec)) != 0) {
|
||||
uWarn("%s:%d failed to convert %s to int64 since %s", __func__, __LINE__, pSec + 1, tstrerror(code));
|
||||
continue;
|
||||
}
|
||||
if (fileSec <= 100) {
|
||||
continue;
|
||||
}
|
||||
|
||||
char fullName[PATH_MAX] = {0};
|
||||
snprintf(fullName, sizeof(fullName), "%s%s%s", tsLogDir, TD_DIRSEP, fname);
|
||||
|
||||
int64_t mtime = 0;
|
||||
if ((code = taosStatFile(fullName, NULL, &mtime, NULL)) != 0) {
|
||||
uWarn("%s:%d failed to stat file %s since %s", __func__, __LINE__, fullName, tstrerror(code));
|
||||
continue;
|
||||
}
|
||||
|
||||
int64_t inactiveSec = taosGetTimestampMs() / 1000 - mtime;
|
||||
|
||||
if (inactiveSec < LOG_INACTIVE_TIME) {
|
||||
continue;
|
||||
}
|
||||
|
||||
int32_t days = inactiveSec / 86400 + 1;
|
||||
if (tsLogKeepDays > 0 && days > tsLogKeepDays) {
|
||||
TAOS_UNUSED(taosRemoveFile(fullName));
|
||||
uInfo("file:%s is removed, days:%d, keepDays:%d, sed:%" PRId64, fullName, days, tsLogKeepDays, fileSec);
|
||||
} else {
|
||||
taosKeepOldLog(fullName); // compress
|
||||
}
|
||||
}
|
||||
if ((code = taosCloseDir(&pDir)) != 0) {
|
||||
uWarn("%s:%d failed to close dir %s since %s\n", __func__, __LINE__, tsLogDir, tstrerror(code));
|
||||
}
|
||||
|
||||
if (tsLogKeepDays > 0) {
|
||||
taosRemoveOldFiles(tsLogDir, tsLogKeepDays);
|
||||
}
|
||||
_exit:
|
||||
taosWUnLockLatch(&tsLogRotateLatch);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void *taosAsyncOutputLog(void *param) {
|
||||
SLogBuff *pLogBuf = (SLogBuff *)tsLogObj.logHandle;
|
||||
SLogBuff *pSlowBuf = (SLogBuff *)tsLogObj.slowHandle;
|
||||
|
|
@ -1007,6 +1171,7 @@ static void *taosAsyncOutputLog(void *param) {
|
|||
int32_t count = 0;
|
||||
int32_t updateCron = 0;
|
||||
int32_t writeInterval = 0;
|
||||
int64_t lastCheckSec = taosGetTimestampMs() / 1000 - (LOG_ROTATE_INTERVAL - LOG_ROTATE_BOOT);
|
||||
|
||||
while (1) {
|
||||
if (pSlowBuf) {
|
||||
|
|
@ -1032,6 +1197,26 @@ static void *taosAsyncOutputLog(void *param) {
|
|||
if (pSlowBuf) taosWriteSlowLog(pSlowBuf);
|
||||
break;
|
||||
}
|
||||
|
||||
// process the log rotation every LOG_ROTATE_INTERVAL
|
||||
int64_t curSec = taosGetTimestampMs() / 1000;
|
||||
if (curSec >= lastCheckSec) {
|
||||
if ((curSec - lastCheckSec) >= LOG_ROTATE_INTERVAL) {
|
||||
TdThread thread;
|
||||
TdThreadAttr attr;
|
||||
(void)taosThreadAttrInit(&attr);
|
||||
(void)taosThreadAttrSetDetachState(&attr, PTHREAD_CREATE_DETACHED);
|
||||
if (taosThreadCreate(&thread, &attr, taosLogRotateFunc, tsLogObj.logHandle) == 0) {
|
||||
uInfo("process log rotation");
|
||||
lastCheckSec = curSec;
|
||||
} else {
|
||||
uWarn("failed to create thread to process log rotation");
|
||||
}
|
||||
(void)taosThreadAttrDestroy(&attr);
|
||||
}
|
||||
} else if (curSec < lastCheckSec) {
|
||||
lastCheckSec = curSec;
|
||||
}
|
||||
}
|
||||
|
||||
return NULL;
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
#define _DEFAULT_SOURCE
|
||||
#include "tutil.h"
|
||||
#include "tlog.h"
|
||||
#include "regex.h"
|
||||
|
||||
void *tmemmem(const char *haystack, int32_t hlen, const char *needle, int32_t nlen) {
|
||||
const char *limit;
|
||||
|
|
@ -520,3 +521,29 @@ int32_t parseCfgReal(const char *str, float *out) {
|
|||
*out = val;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
bool tIsValidFileName(const char *fileName, const char *pattern) {
|
||||
const char *fileNamePattern = "^[a-zA-Z0-9_.-]+$";
|
||||
|
||||
regex_t fileNameReg;
|
||||
|
||||
if (pattern) fileNamePattern = pattern;
|
||||
|
||||
if (regcomp(&fileNameReg, fileNamePattern, REG_EXTENDED) != 0) {
|
||||
fprintf(stderr, "failed to compile file name pattern:%s\n", fileNamePattern);
|
||||
return false;
|
||||
}
|
||||
|
||||
int32_t code = regexec(&fileNameReg, fileName, 0, NULL, 0);
|
||||
regfree(&fileNameReg);
|
||||
if (code != 0) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool tIsValidFilePath(const char *filePath, const char *pattern) {
|
||||
const char *filePathPattern = "^[a-zA-Z0-9:/\\_.-]+$";
|
||||
|
||||
return tIsValidFileName(filePath, pattern ? pattern : filePathPattern);
|
||||
}
|
||||
|
|
@ -0,0 +1,181 @@
|
|||
###################################################################
|
||||
# Copyright (c) 2016 by TAOS Technologies, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is proprietary and confidential to TAOS Technologies.
|
||||
# No part of this file may be reproduced, stored, transmitted,
|
||||
# disclosed or used in any form or by any means other than as
|
||||
# expressly provided by the written permission from Jianhui Tao
|
||||
#
|
||||
###################################################################
|
||||
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys
|
||||
import time
|
||||
import random
|
||||
import taos
|
||||
|
||||
import frame
|
||||
import frame.eos
|
||||
import frame.etime
|
||||
import frame.etool
|
||||
from frame.log import *
|
||||
from frame.sql import *
|
||||
from frame.cases import *
|
||||
from frame.caseBase import *
|
||||
from frame.srvCtl import *
|
||||
from frame import *
|
||||
|
||||
|
||||
class TDTestCase(TBase):
|
||||
# parse line
|
||||
def parseLine(self, line):
|
||||
line = line.strip()
|
||||
PRE_DEFINE = "#define TSDB_CODE_"
|
||||
n = len(PRE_DEFINE)
|
||||
if line[:n] != PRE_DEFINE:
|
||||
return None
|
||||
# TAOS_DEF_ERROR_CODE(0, 0x000B)
|
||||
pos = line.find("TAOS_DEF_ERROR_CODE(0, 0x", n)
|
||||
if pos == -1:
|
||||
tdLog.info(f"not found \"TAOS_DEF_ERROR_CODE(0, \" line={line}")
|
||||
return None
|
||||
|
||||
code = line[pos:].strip()
|
||||
pos = code.find(")")
|
||||
if pos == -1:
|
||||
tdLog.info(f"not found \")\", line={line}")
|
||||
return None
|
||||
code = code[:pos]
|
||||
if len(code) != 4:
|
||||
tdLog.info(f"code is len not 4 len:{len(code)} subcode={code}\")\", line={line}")
|
||||
return None
|
||||
|
||||
# return
|
||||
return "0x8000" + code
|
||||
|
||||
# ignore error
|
||||
def ignoreCode(self, code):
|
||||
ignoreCodes = {"0x00008, 0x000009"}
|
||||
if code in ignoreCodes:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
# read valid code
|
||||
def readHeadCodes(self, hFile):
|
||||
codes = []
|
||||
start = False
|
||||
# read
|
||||
with open(hFile) as file:
|
||||
for line in file:
|
||||
code = self.parseLine(line)
|
||||
# invalid
|
||||
if code == None:
|
||||
continue
|
||||
# ignore
|
||||
if self.ignoreCode(code):
|
||||
tdLog.info(f"ignore error {code}\n")
|
||||
# valid
|
||||
if code == 0:
|
||||
start = True
|
||||
if start:
|
||||
codes.append(code)
|
||||
# return
|
||||
return codes
|
||||
|
||||
# parse doc lines
|
||||
def parseDocLine(self, line):
|
||||
line = line.strip()
|
||||
PRE_DEFINE = "| 0x8000"
|
||||
n = len(PRE_DEFINE)
|
||||
if line[:n] != PRE_DEFINE:
|
||||
return None
|
||||
line = line[2:]
|
||||
cols = line.split("|")
|
||||
# remove blank
|
||||
cols = [col.strip() for col in cols]
|
||||
|
||||
# return
|
||||
return cols
|
||||
|
||||
|
||||
# read valid code
|
||||
def readDocCodes(self, docFile):
|
||||
codes = []
|
||||
start = False
|
||||
# read
|
||||
with open(docFile) as file:
|
||||
for line in file:
|
||||
code = self.parseDocLine(line)
|
||||
# invalid
|
||||
if code == None:
|
||||
continue
|
||||
# valid
|
||||
if start:
|
||||
codes.append(code)
|
||||
# return
|
||||
return codes
|
||||
|
||||
# check
|
||||
def checkConsistency(self, docCodes, codes):
|
||||
diff = False
|
||||
# len
|
||||
docLen = len(docCodes)
|
||||
len = len(codes)
|
||||
tdLog.info("head file codes = {len} doc file codes={docLen} \n")
|
||||
|
||||
if docLen > len:
|
||||
maxLen = docLen
|
||||
else:
|
||||
maxLen = len
|
||||
|
||||
for i in range(maxLen):
|
||||
if i < len and i < docLen:
|
||||
if codes[i] == docCodes[i][0]:
|
||||
tdLog.info(f" i={i} same head code: {codes[i]} doc code:{docCodes[i][0]}\n")
|
||||
else:
|
||||
tdLog.info(f" i={i} diff head code: {codes[i]} doc code:{docCodes[i][0]}\n")
|
||||
diff = True
|
||||
elif i < len:
|
||||
tdLog.info(f" i={i} diff head code: {codes[i]} doc code: None\n")
|
||||
diff = True
|
||||
elif i < docLen:
|
||||
tdLog.info(f" i={i} diff head code: None doc code: {docCodes[i][0]}\n")
|
||||
diff = True
|
||||
|
||||
# result
|
||||
if diff:
|
||||
tdLog.exit("check error code consistency failed.\n")
|
||||
|
||||
|
||||
# run
|
||||
def run(self):
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
|
||||
# read head error code
|
||||
hFile = "../../include/util/taoserror.h"
|
||||
codes = self.readHeadCodes(hFile)
|
||||
|
||||
# read zh codes
|
||||
zhDoc = "../../docs/zh/14-reference/09-error-code.md"
|
||||
zhCodes = self.readDocCodes(zhDoc, codes)
|
||||
|
||||
# read en codes
|
||||
enDoc = "../../docs/en/14-reference/09-error-code.md"
|
||||
enCodes = self.readDocCodes(enDoc, codes)
|
||||
|
||||
# check zh
|
||||
tdLog.info(f"check zh docs ...\n")
|
||||
self.checkConsistency(zhCodes, codes)
|
||||
|
||||
# check en
|
||||
tdLog.info(f"check en docs ...\n")
|
||||
self.checkConsistency(enCodes, codes)
|
||||
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
|
|
@ -0,0 +1,241 @@
|
|||
###################################################################
|
||||
# Copyright (c) 2016 by TAOS Technologies, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is proprietary and confidential to TAOS Technologies.
|
||||
# No part of this file may be reproduced, stored, transmitted,
|
||||
# disclosed or used in any form or by any means other than as
|
||||
# expressly provided by the written permission from Jianhui Tao
|
||||
#
|
||||
###################################################################
|
||||
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys
|
||||
import time
|
||||
import random
|
||||
import taos
|
||||
|
||||
import frame
|
||||
import frame.eos
|
||||
import frame.etime
|
||||
import frame.etool
|
||||
|
||||
from frame.log import *
|
||||
from frame.sql import *
|
||||
from frame.cases import *
|
||||
from frame.caseBase import *
|
||||
from frame.srvCtl import *
|
||||
from frame import *
|
||||
|
||||
ignoreCodes = [
|
||||
'0x80000023', '0x80000024', '0x80000025', '0x80000026', '0x80000027', '0x80000028', '0x80000029', '0x8000002A', '0x80000109', '0x80000110',
|
||||
'0x80000129', '0x8000012C', '0x8000012D', '0x8000012E', '0x8000012F', '0x80000136', '0x80000137', '0x80000138', '0x80000139', '0x8000013A',
|
||||
'0x8000013B', '0x80000200', '0x80000201', '0x80000202', '0x80000203', '0x80000204', '0x80000205', '0x80000206', '0x8000020B', '0x8000020E',
|
||||
'0x80000210', '0x80000212', '0x80000214', '0x80000215', '0x80000217', '0x8000021B', '0x8000021C', '0x8000021D', '0x8000021E', '0x80000220',
|
||||
'0x80000221', '0x8000022C', '0x80000232', '0x80000233', '0x80000234', '0x80000235', '0x80000236', '0x800002FF', '0x80000300', '0x80000316',
|
||||
'0x80000317', '0x80000338', '0x80000339', '0x8000033F', '0x80000343', '0x80000345', '0x80000356', '0x80000359', '0x8000035A', '0x8000035B',
|
||||
'0x8000035C', '0x8000035D', '0x80000362', '0x8000038C', '0x8000038E', '0x8000039B', '0x8000039C', '0x8000039D', '0x8000039E', '0x800003A6',
|
||||
'0x800003A7', '0x800003AA', '0x800003AB', '0x800003AC', '0x800003AD', '0x800003B0', '0x800003B2', '0x800003B4', '0x800003B5', '0x800003BA',
|
||||
'0x800003C0', '0x800003C1', '0x800003D0', '0x800003D8', '0x800003D9', '0x800003E2', '0x800003F4', '0x800003F8', '0x80000412', '0x80000413',
|
||||
'0x80000414', '0x80000415', '0x80000416', '0x80000417', '0x80000418', '0x80000419', '0x80000420', '0x80000421', '0x80000422', '0x80000423',
|
||||
'0x80000424', '0x80000425', '0x80000426', '0x80000427', '0x80000428', '0x80000429', '0x8000042A', '0x80000430', '0x80000431', '0x80000432',
|
||||
'0x80000433', '0x80000434', '0x80000435', '0x80000436', '0x80000437', '0x80000438', '0x80000440', '0x80000441', '0x80000442', '0x80000443',
|
||||
'0x80000444', '0x80000445', '0x80000446', '0x80000447', '0x80000485', '0x80000486', '0x800004A0', '0x800004A1', '0x800004B1', '0x800004B2',
|
||||
'0x800004B3', '0x80000504', '0x80000528', '0x80000532', '0x80000533', '0x80000534', '0x80000535', '0x80000536', '0x80000537', '0x80000538',
|
||||
'0x80000539', '0x80000601', '0x80000607', '0x8000060A', '0x8000060D', '0x8000060E', '0x8000060F', '0x80000610', '0x80000611', '0x80000612',
|
||||
'0x80000613', '0x80000614', '0x80000615', '0x80000616', '0x8000061C', '0x8000061E', '0x80000701', '0x80000705', '0x80000706', '0x80000707',
|
||||
'0x80000708', '0x8000070B', '0x8000070C', '0x8000070E', '0x8000070F', '0x80000712', '0x80000723', '0x80000724', '0x80000725', '0x80000726',
|
||||
'0x80000727', '0x80000728', '0x8000072A', '0x8000072C', '0x8000072D', '0x8000072E', '0x80000730', '0x80000731', '0x80000732', '0x80000733',
|
||||
'0x80000734', '0x80000735', '0x80000736', '0x80000737', '0x80000738', '0x8000080E', '0x8000080F', '0x80000810', '0x80000811', '0x80000812',
|
||||
'0x80000813', '0x80000814', '0x80000815', '0x80000816', '0x80000817', '0x80000818', '0x80000819', '0x80000820', '0x80000821', '0x80000822',
|
||||
'0x80000823', '0x80000824', '0x80000825', '0x80000826', '0x80000827', '0x80000828', '0x80000829', '0x8000082A', '0x8000082B', '0x8000082C',
|
||||
'0x8000082D', '0x80000907', '0x80000919', '0x8000091A', '0x8000091B', '0x8000091C', '0x8000091D', '0x8000091E', '0x8000091F', '0x80000A00',
|
||||
'0x80000A01', '0x80000A03', '0x80000A06', '0x80000A07', '0x80000A08', '0x80000A09', '0x80000A0A', '0x80000A0B', '0x80000A0E', '0x80002206',
|
||||
'0x80002207', '0x80002406', '0x80002407', '0x80002503', '0x80002506', '0x80002507', '0x8000261B', '0x80002653', '0x80002668', '0x80002669',
|
||||
'0x8000266A', '0x8000266B', '0x8000266C', '0x8000266D', '0x8000266E', '0x8000266F', '0x80002670', '0x80002671', '0x80002672', '0x80002673',
|
||||
'0x80002674', '0x80002675', '0x80002676', '0x80002677', '0x80002678', '0x80002679', '0x8000267A', '0x8000267B', '0x8000267C', '0x8000267D',
|
||||
'0x8000267E', '0x8000267F', '0x80002680', '0x80002681', '0x80002682', '0x80002683', '0x80002684', '0x80002685', '0x80002703', '0x80002806',
|
||||
'0x80002807', '0x80002808', '0x80002809', '0x8000280A', '0x8000280B', '0x8000280C', '0x8000280D', '0x8000280E', '0x8000280F', '0x80002810',
|
||||
'0x80002811', '0x80002812', '0x80002813', '0x80002814', '0x80002815', '0x8000290B', '0x80002920', '0x80003003', '0x80003006', '0x80003106',
|
||||
'0x80003107', '0x80003108', '0x80003109', '0x80003110', '0x80003111', '0x80003112', '0x80003250', '0x80004003', '0x80004004', '0x80004005',
|
||||
'0x80004006', '0x80004007', '0x80004008', '0x80004009', '0x80004010', '0x80004011', '0x80004012', '0x80004013', '0x80004014', '0x80004015',
|
||||
'0x80004016', '0x80004102', '0x80004103', '0x80004104', '0x80004105', '0x80004106', '0x80004107', '0x80004108', '0x80004109', '0x80005100',
|
||||
'0x80005101', '0x80006000', '0x80006100', '0x80006101', '0x80006102', '0x80000019', '0x80002639', '0x80002666']
|
||||
|
||||
|
||||
class TDTestCase(TBase):
|
||||
# parse line
|
||||
def parseLine(self, line, show):
|
||||
line = line.strip()
|
||||
# PRE_DEFINE
|
||||
PRE_DEFINE = "#define TSDB_CODE_"
|
||||
n = len(PRE_DEFINE)
|
||||
if line[:n] != PRE_DEFINE:
|
||||
return None
|
||||
# MID_DEFINE
|
||||
MID_DEFINE = "TAOS_DEF_ERROR_CODE(0, 0x"
|
||||
pos = line.find(MID_DEFINE, n)
|
||||
if pos == -1:
|
||||
if show:
|
||||
tdLog.info(f"not found \"{MID_DEFINE}\" line={line}")
|
||||
return None
|
||||
start = pos + len(MID_DEFINE)
|
||||
code = line[start:].strip()
|
||||
# )
|
||||
pos = code.find(")")
|
||||
if pos == -1:
|
||||
if show:
|
||||
tdLog.info(f"not found \")\", code={code}")
|
||||
return None
|
||||
# check len
|
||||
code = code[:pos]
|
||||
if len(code) != 4:
|
||||
if show:
|
||||
tdLog.info(f"code is len not 4 len:{len(code)} subcode={code}\")\", line={line}")
|
||||
return None
|
||||
|
||||
# return
|
||||
return "0x8000" + code
|
||||
|
||||
# ignore error
|
||||
def ignoreCode(self, code):
|
||||
if code in ignoreCodes:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
# read valid code
|
||||
def readHeadCodes(self, hFile):
|
||||
codes = []
|
||||
start = False
|
||||
# read
|
||||
with open(hFile) as file:
|
||||
for line in file:
|
||||
code = self.parseLine(line, start)
|
||||
# invalid
|
||||
if code == None:
|
||||
continue
|
||||
#print(code)
|
||||
# valid
|
||||
if code == "0x8000000B":
|
||||
start = True
|
||||
if start:
|
||||
codes.append(code)
|
||||
# return
|
||||
return codes
|
||||
|
||||
# parse doc lines
|
||||
def parseDocLine(self, line):
|
||||
line = line.strip()
|
||||
PRE_DEFINE = "| 0x8000"
|
||||
n = len(PRE_DEFINE)
|
||||
if line[:n] != PRE_DEFINE:
|
||||
return None
|
||||
line = line[2:]
|
||||
cols = line.split("|")
|
||||
# remove blank
|
||||
cols = [col.strip() for col in cols]
|
||||
#print(cols)
|
||||
|
||||
# return
|
||||
return cols
|
||||
|
||||
|
||||
# read valid code
|
||||
def readDocCodes(self, docFile):
|
||||
codes = []
|
||||
start = False
|
||||
# read
|
||||
with open(docFile) as file:
|
||||
for line in file:
|
||||
code = self.parseDocLine(line)
|
||||
# invalid
|
||||
if code == None:
|
||||
continue
|
||||
# valid
|
||||
codes.append(code)
|
||||
|
||||
# return
|
||||
return codes
|
||||
|
||||
# check
|
||||
def checkConsistency(self, docCodes, codes, checkCol2=True, checkCol3=True):
|
||||
failed = 0
|
||||
ignored = 0
|
||||
# len
|
||||
hLen = len(codes)
|
||||
docLen = len(docCodes)
|
||||
#errCodes = []
|
||||
#tdLog.info(f"head file codes items= {hLen} doc file codes items={docLen} \n")
|
||||
for i in range(hLen):
|
||||
problem = True
|
||||
action = "not found"
|
||||
for j in range(docLen):
|
||||
if codes[i] == docCodes[j][0]:
|
||||
#tdLog.info(f" i={i} error code: {codes[i]} found in doc code:{docCodes[j][0]}\n")
|
||||
problem = False
|
||||
if docCodes[j][1] == "":
|
||||
# describe col1 must not empty
|
||||
problem = True
|
||||
action = "found but \"Error Description\" column is empty"
|
||||
# check col2 empty
|
||||
elif checkCol2 and docCodes[j][2] == "":
|
||||
problem = True
|
||||
action = "found but \"Possible Cause\" column is empty"
|
||||
# check col3 empty
|
||||
elif checkCol3 and docCodes[j][3] == "":
|
||||
problem = True
|
||||
action = "found but \"Suggested Actions\" column is empty"
|
||||
|
||||
# found stop search next
|
||||
break
|
||||
|
||||
if problem:
|
||||
# ignore
|
||||
if self.ignoreCode(codes[i]):
|
||||
#tdLog.info(f" i={i} error code: {codes[i]} ignore ...")
|
||||
ignored += 1
|
||||
else:
|
||||
tdLog.info(f" i={i} error code: {codes[i]} {action} in doc.")
|
||||
failed +=1
|
||||
#errCodes.append(codes[i])
|
||||
|
||||
#print(errCodes)
|
||||
# result
|
||||
if failed > 0:
|
||||
tdLog.exit("Failed to check the consistency of error codes between header and doc. "
|
||||
f"failed:{failed}, ignored:{ignored}, all:{hLen}\n")
|
||||
|
||||
tdLog.info(f"Check consistency successfully. ok items={hLen - ignored}, ignored items={ignored}\n")
|
||||
|
||||
|
||||
# run
|
||||
def run(self):
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
|
||||
# read head error code
|
||||
hFile = "../../include/util/taoserror.h"
|
||||
codes = self.readHeadCodes(hFile)
|
||||
|
||||
# read zh codes
|
||||
zhDoc = "../../docs/zh/14-reference/09-error-code.md"
|
||||
zhCodes = self.readDocCodes(zhDoc)
|
||||
|
||||
# read en codes
|
||||
enDoc = "../../docs/en/14-reference/09-error-code.md"
|
||||
enCodes = self.readDocCodes(enDoc)
|
||||
|
||||
# check zh
|
||||
tdLog.info(f"check zh docs ...")
|
||||
self.checkConsistency(zhCodes, codes)
|
||||
|
||||
# check en
|
||||
tdLog.info(f"check en docs ...")
|
||||
self.checkConsistency(enCodes, codes)
|
||||
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
|
|
@ -53,6 +53,7 @@
|
|||
,,y,army,./pytest.sh python3 ./test.py -f query/test_having.py
|
||||
,,n,army,python3 ./test.py -f tmq/drop_lost_comsumers.py
|
||||
,,y,army,./pytest.sh python3 ./test.py -f cmdline/taosCli.py
|
||||
,,n,army,python3 ./test.py -f whole/checkErrorCode.py
|
||||
|
||||
#
|
||||
# system test
|
||||
|
|
@ -354,6 +355,7 @@
|
|||
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/telemetry.py
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/backquote_check.py
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdMonitor.py
|
||||
,,n,system-test,python3 ./test.py -f 0-others/taosdlog.py
|
||||
,,n,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3
|
||||
,,n,system-test,python3 ./test.py -f 0-others/udfTest.py
|
||||
,,n,system-test,python3 ./test.py -f 0-others/udf_create.py
|
||||
|
|
|
|||
|
|
@ -1,66 +1,351 @@
|
|||
import taos
|
||||
import sys
|
||||
import time
|
||||
import os
|
||||
|
||||
from util.log import *
|
||||
from util.sql import *
|
||||
from util.cases import *
|
||||
from util.dnodes import *
|
||||
|
||||
class TDTestCase:
|
||||
|
||||
def init(self, conn, logSql, replicaVar=1):
|
||||
self.replicaVar = int(replicaVar)
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
tdSql.init(conn.cursor())
|
||||
|
||||
def getBuildPath(self):
|
||||
selfPath = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
if ("community" in selfPath):
|
||||
projPath = selfPath[:selfPath.find("community")]
|
||||
else:
|
||||
projPath = selfPath[:selfPath.find("tests")]
|
||||
|
||||
for root, dirs, files in os.walk(projPath):
|
||||
if ("taosd" in files or "taosd.exe" in files):
|
||||
rootRealPath = os.path.dirname(os.path.realpath(root))
|
||||
if ("packaging" not in rootRealPath):
|
||||
buildPath = root[:len(root) - len("/build/bin")]
|
||||
break
|
||||
return buildPath
|
||||
|
||||
def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
|
||||
tdSql.prepare()
|
||||
# time.sleep(2)
|
||||
tdSql.query("create user testpy pass 'testpy'")
|
||||
|
||||
buildPath = self.getBuildPath()
|
||||
if (buildPath == ""):
|
||||
tdLog.exit("taosd not found!")
|
||||
else:
|
||||
tdLog.info("taosd found in %s" % buildPath)
|
||||
logPath = buildPath + "/../sim/dnode1/log"
|
||||
tdLog.info("log path: %s" % logPath)
|
||||
|
||||
tdDnodes.stop(1)
|
||||
time.sleep(2)
|
||||
tdSql.error("select * from information_schema.ins_databases")
|
||||
os.system("rm -rf %s" % logPath)
|
||||
if os.path.exists(logPath) == True:
|
||||
tdLog.exit("log pat still exist!")
|
||||
|
||||
tdDnodes.start(1)
|
||||
time.sleep(2)
|
||||
if os.path.exists(logPath) != True:
|
||||
tdLog.exit("log pat is not generated!")
|
||||
|
||||
tdSql.query("select * from information_schema.ins_databases")
|
||||
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
import concurrent.futures
|
||||
import os
|
||||
import os.path
|
||||
import platform
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from util.log import *
|
||||
from util.sql import *
|
||||
from util.cases import *
|
||||
from util.dnodes import *
|
||||
|
||||
class TDTestCase:
|
||||
|
||||
def init(self, conn, logSql, replicaVar=1):
|
||||
self.replicaVar = int(replicaVar)
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
tdSql.init(conn.cursor())
|
||||
tdSql.prepare()
|
||||
self.buildPath = self.getBuildPath()
|
||||
if (self.buildPath == ""):
|
||||
tdLog.exit("taosd not found!")
|
||||
else:
|
||||
tdLog.info("taosd found in %s" % self.buildPath)
|
||||
self.taosdPath = self.buildPath + "/build/bin/taosd"
|
||||
self.taosPath = self.buildPath + "/build/bin/taos"
|
||||
self.logPath = self.buildPath + "/../sim/dnode1/log"
|
||||
tdLog.info("taosd path: %s" % self.taosdPath)
|
||||
tdLog.info("taos path: %s" % self.taosPath)
|
||||
tdLog.info("log path: %s" % self.logPath)
|
||||
self.commonCfgDict = {}
|
||||
|
||||
def getBuildPath(self):
|
||||
selfPath = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
if ("community" in selfPath):
|
||||
projPath = selfPath[:selfPath.find("community")]
|
||||
else:
|
||||
projPath = selfPath[:selfPath.find("tests")]
|
||||
|
||||
for root, dirs, files in os.walk(projPath):
|
||||
if ("taosd" in files or "taosd.exe" in files):
|
||||
rootRealPath = os.path.dirname(os.path.realpath(root))
|
||||
if ("packaging" not in rootRealPath):
|
||||
buildPath = root[:len(root) - len("/build/bin")]
|
||||
break
|
||||
return buildPath
|
||||
|
||||
def prepareCfg(self, cfgPath, cfgDict):
|
||||
tdLog.info("make dir %s" % cfgPath)
|
||||
os.makedirs(cfgPath, exist_ok=True)
|
||||
with open(cfgPath + "/taos.cfg", "w") as f:
|
||||
for key in self.commonCfgDict:
|
||||
f.write("%s %s\n" % (key, self.commonCfgDict[key]))
|
||||
for key in cfgDict:
|
||||
f.write("%s %s\n" % (key, cfgDict[key]))
|
||||
if not os.path.exists(cfgPath + "/taos.cfg"):
|
||||
tdLog.exit("taos.cfg not found in %s" % cfgPath)
|
||||
else:
|
||||
tdLog.info("taos.cfg found in %s" % cfgPath)
|
||||
|
||||
def closeBin(self, binName):
|
||||
tdLog.info("Closing %s" % binName)
|
||||
if platform.system().lower() == 'windows':
|
||||
psCmd = "for /f %%a in ('wmic process where \"name='%s.exe'\" get processId ^| xargs echo ^| awk ^'{print $2}^' ^&^& echo aa') do @(ps | grep %%a | awk '{print $1}' | xargs)" % binName
|
||||
else:
|
||||
psCmd = "ps -ef | grep -w %s | grep -v grep | awk '{print $2}'" % binName
|
||||
tdLog.info(f"psCmd:{psCmd}")
|
||||
|
||||
try:
|
||||
rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
processID = rem.stdout.decode().strip()
|
||||
tdLog.info(f"processID:{processID}")
|
||||
except Exception as e:
|
||||
tdLog.info(f"closeBin error:{e}")
|
||||
processID = ""
|
||||
onlyKillOnceWindows = 0
|
||||
while(processID):
|
||||
if not platform.system().lower() == 'windows' or (onlyKillOnceWindows == 0 and platform.system().lower() == 'windows'):
|
||||
killCmd = "kill -9 %s > /dev/null 2>&1" % processID
|
||||
if platform.system().lower() == 'windows':
|
||||
killCmd = "kill -9 %s > nul 2>&1" % processID
|
||||
tdLog.info(f"killCmd:{killCmd}")
|
||||
os.system(killCmd)
|
||||
tdLog.info(f"killed {binName} process {processID}")
|
||||
onlyKillOnceWindows = 1
|
||||
time.sleep(1)
|
||||
try:
|
||||
rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
processID = rem.stdout.decode().strip()
|
||||
except Exception as e:
|
||||
tdLog.info(f"closeBin error:{e}")
|
||||
processID = ""
|
||||
|
||||
def closeTaosd(self, signal=9):
|
||||
self.closeBin("taosd")
|
||||
|
||||
def closeTaos(self, signal=9):
|
||||
self.closeBin("taos")
|
||||
|
||||
def openBin(self, binPath, waitSec=5):
|
||||
tdLog.info(f"Opening {binPath}")
|
||||
try:
|
||||
process = subprocess.Popen(binPath, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
time.sleep(waitSec)
|
||||
if process.poll() is None:
|
||||
tdLog.info(f"{binPath} started successfully")
|
||||
else:
|
||||
error = process.stderr.read().decode(encoding="utf-8").strip()
|
||||
raise Exception(f"Failed to start {binPath}: {error}")
|
||||
except Exception as e:
|
||||
raise Exception(f"Failed to start {binPath}: %s" % repr(e))
|
||||
|
||||
def openTaosd(self, args="", waitSec=8):
|
||||
self.openBin(f'{self.taosdPath} {args}', waitSec)
|
||||
|
||||
def openTaos(self, args="", waitSec=5):
|
||||
self.openBin(f'{self.taosPath} {args}', waitSec)
|
||||
|
||||
|
||||
def prepare_logoutput(self, desc, port, logOutput, skipOpenBin=False):
|
||||
tdLog.info("Preparing %s, port:%s" % (desc, port))
|
||||
dnodePath = self.buildPath + "/../sim/dnode%s" % port
|
||||
tdLog.info("remove dnodePath:%s" % dnodePath)
|
||||
try:
|
||||
if os.path.exists(dnodePath):
|
||||
shutil.rmtree(dnodePath)
|
||||
except Exception as e:
|
||||
raise Exception(f"Failed to remove directory {dnodePath}: {e}")
|
||||
try:
|
||||
self.prepareCfg(dnodePath, {"serverPort": port,
|
||||
"dataDir": dnodePath + os.sep + "data",
|
||||
"logDir": dnodePath + os.sep + "log"})
|
||||
except Exception as e:
|
||||
raise Exception(f"Failed to prepare configuration for {dnodePath}: {e}")
|
||||
try:
|
||||
self.openTaosd(f"-c {dnodePath} {logOutput}")
|
||||
self.openTaos(f"-c {dnodePath} {logOutput}")
|
||||
except Exception as e:
|
||||
if(skipOpenBin):
|
||||
tdLog.info(f"Failed to prepare taosd and taos with log output {logOutput}: {e}")
|
||||
else:
|
||||
raise Exception(f"Failed to prepare taosd and taos with log output {logOutput}: {e}")
|
||||
|
||||
|
||||
def prepare_stdout(self):
|
||||
list = self.prepare_list[0]
|
||||
self.prepare_logoutput(list[0], list[1], "-o " + list[0])
|
||||
|
||||
def check_stdout(self):
|
||||
tdLog.info("Running check stdout")
|
||||
dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[0][1]
|
||||
tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0"))
|
||||
tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0"))
|
||||
|
||||
def prepare_stderr(self):
|
||||
list = self.prepare_list[1]
|
||||
self.prepare_logoutput(list[0], list[1], "--log-output " + list[0])
|
||||
|
||||
def check_stderr(self):
|
||||
tdLog.info("Running check stderr")
|
||||
dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[1][1]
|
||||
tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0"))
|
||||
tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0"))
|
||||
|
||||
def prepare_dev_null(self):
|
||||
list = self.prepare_list[2]
|
||||
# self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0])
|
||||
|
||||
def check_dev_null(self):
|
||||
tdLog.info("Running check /dev/null")
|
||||
# dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[2][1]
|
||||
# tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0"))
|
||||
# tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0"))
|
||||
|
||||
def prepare_fullpath(self):
|
||||
list = self.prepare_list[3]
|
||||
dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[3][1]
|
||||
self.prepare_logoutput(list[0], list[1], "-o " + dnodePath + "/log0/" )
|
||||
|
||||
def check_fullpath(self):
|
||||
tdLog.info("Running check fullpath")
|
||||
logPath = self.buildPath + "/../sim/dnode%s/log0/" % self.prepare_list[3][1]
|
||||
tdSql.checkEqual(True, os.path.exists(f"{logPath}taosdlog.0"))
|
||||
tdSql.checkEqual(True, os.path.exists(f"{logPath}taoslog0.0"))
|
||||
|
||||
def prepare_fullname(self):
|
||||
list = self.prepare_list[4]
|
||||
dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[4][1]
|
||||
self.prepare_logoutput(list[0], list[1], "--log-output " + dnodePath + "/log0/" + list[0])
|
||||
dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[4][1]
|
||||
|
||||
def check_fullname(self):
|
||||
tdLog.info("Running check fullname")
|
||||
logPath = self.buildPath + "/../sim/dnode%s/log0/" % self.prepare_list[4][1]
|
||||
tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[4][0] + ".0"))
|
||||
tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[4][0] + "0.0"))
|
||||
|
||||
def prepare_relativepath(self):
|
||||
list = self.prepare_list[5]
|
||||
self.prepare_logoutput(list[0], list[1], "--log-output=" + "log0/")
|
||||
|
||||
def check_relativepath(self):
|
||||
tdLog.info("Running check relativepath")
|
||||
logPath = self.buildPath + "/../sim/dnode%s/log/log0/" % self.prepare_list[5][1]
|
||||
tdSql.checkEqual(True, os.path.exists(logPath + "taosdlog.0"))
|
||||
tdSql.checkEqual(True, os.path.exists(logPath + "taoslog0.0"))
|
||||
|
||||
def prepare_relativename(self):
|
||||
list = self.prepare_list[6]
|
||||
self.prepare_logoutput(list[0], list[1], "-o " + "log0/" + list[0])
|
||||
def check_relativename(self):
|
||||
tdLog.info("Running check relativename")
|
||||
logPath = self.buildPath + "/../sim/dnode%s/log/log0/" % self.prepare_list[6][1]
|
||||
tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[6][0] + ".0"))
|
||||
tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[6][0] + "0.0"))
|
||||
|
||||
def prepare_filename(self):
|
||||
list = self.prepare_list[7]
|
||||
self.prepare_logoutput(list[0], list[1], "--log-output " + list[0])
|
||||
def check_filename(self):
|
||||
tdLog.info("Running check filename")
|
||||
logPath = self.buildPath + "/../sim/dnode%s/log/" % self.prepare_list[7][1]
|
||||
tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[7][0] + ".0"))
|
||||
tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[7][0] + "0.0"))
|
||||
|
||||
def prepare_empty(self):
|
||||
list = self.prepare_list[8]
|
||||
self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0], True)
|
||||
def check_empty(self):
|
||||
tdLog.info("Running check empty")
|
||||
logPath = self.buildPath + "/../sim/dnode%s/log" % self.prepare_list[8][1]
|
||||
tdSql.checkEqual(False, os.path.exists(f"{logPath}/taosdlog.0"))
|
||||
tdSql.checkEqual(False, os.path.exists(f"{logPath}/taoslog.0"))
|
||||
|
||||
def prepare_illegal(self):
|
||||
list = self.prepare_list[9]
|
||||
self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0], True)
|
||||
def check_illegal(self):
|
||||
tdLog.info("Running check empty")
|
||||
logPath = self.buildPath + "/../sim/dnode%s/log" % self.prepare_list[9][1]
|
||||
tdSql.checkEqual(False, os.path.exists(f"{logPath}/taosdlog.0"))
|
||||
tdSql.checkEqual(False, os.path.exists(f"{logPath}/taoslog.0"))
|
||||
|
||||
def prepareCheckResources(self):
|
||||
self.prepare_list = [["stdout", "10030"], ["stderr", "10031"], ["/dev/null", "10032"], ["fullpath", "10033"],
|
||||
["fullname", "10034"], ["relativepath", "10035"], ["relativename", "10036"], ["filename", "10037"],
|
||||
["empty", "10038"], ["illeg?al", "10039"]]
|
||||
self.check_functions = {
|
||||
self.prepare_stdout: self.check_stdout,
|
||||
self.prepare_stderr: self.check_stderr,
|
||||
self.prepare_dev_null: self.check_dev_null,
|
||||
self.prepare_fullpath: self.check_fullpath,
|
||||
self.prepare_fullname: self.check_fullname,
|
||||
self.prepare_relativepath: self.check_relativepath,
|
||||
self.prepare_relativename: self.check_relativename,
|
||||
self.prepare_filename: self.check_filename,
|
||||
self.prepare_empty: self.check_empty,
|
||||
self.prepare_illegal: self.check_illegal,
|
||||
}
|
||||
|
||||
def checkLogOutput(self):
|
||||
self.closeTaosd()
|
||||
self.closeTaos()
|
||||
self.prepareCheckResources()
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
prepare_futures = [executor.submit(prepare_func) for prepare_func, _ in self.check_functions.items()]
|
||||
for future in concurrent.futures.as_completed(prepare_futures):
|
||||
try:
|
||||
future.result()
|
||||
except Exception as e:
|
||||
raise Exception(f"Error in prepare function: {e}")
|
||||
|
||||
check_futures = [executor.submit(check_func) for _, check_func in self.check_functions.items()]
|
||||
for future in concurrent.futures.as_completed(check_futures):
|
||||
try:
|
||||
future.result()
|
||||
except Exception as e:
|
||||
raise Exception(f"Error in prepare function: {e}")
|
||||
|
||||
def checkLogRotate(self):
|
||||
tdLog.info("Running check log rotate")
|
||||
dnodePath = self.buildPath + "/../sim/dnode10050"
|
||||
logRotateAfterBoot = 6 # LOG_ROTATE_BOOT
|
||||
self.closeTaosd()
|
||||
self.closeTaos()
|
||||
try:
|
||||
if os.path.exists(dnodePath):
|
||||
shutil.rmtree(dnodePath)
|
||||
self.prepareCfg(dnodePath, {"serverPort": 10050,
|
||||
"dataDir": dnodePath + os.sep + "data",
|
||||
"logDir": dnodePath + os.sep + "log",
|
||||
"logKeepDays": "3" })
|
||||
except Exception as e:
|
||||
raise Exception(f"Failed to prepare configuration for {dnodePath}: {e}")
|
||||
|
||||
nowSec = int(time.time())
|
||||
stubFile99 = f"{dnodePath}/log/taosdlog.99"
|
||||
stubFile101 = f"{dnodePath}/log/taosdlog.101"
|
||||
stubGzFile98 = f"{dnodePath}/log/taosdlog.98.gz"
|
||||
stubGzFile102 = f"{dnodePath}/log/taosdlog.102.gz"
|
||||
stubFileNow = f"{dnodePath}/log/taosdlog.{nowSec}"
|
||||
stubGzFileNow = f"{dnodePath}/log/taosdlog.%d.gz" % (nowSec - 1)
|
||||
stubGzFileKeep = f"{dnodePath}/log/taosdlog.%d.gz" % (nowSec - 86400 * 2)
|
||||
stubGzFileDel = f"{dnodePath}/log/taosdlog.%d.gz" % (nowSec - 86400 * 3)
|
||||
stubFiles = [stubFile99, stubFile101, stubGzFile98, stubGzFile102, stubFileNow, stubGzFileNow, stubGzFileKeep, stubGzFileDel]
|
||||
|
||||
try:
|
||||
os.makedirs(f"{dnodePath}/log", exist_ok=True)
|
||||
for stubFile in stubFiles:
|
||||
with open(stubFile, "w") as f:
|
||||
f.write("test log rotate")
|
||||
except Exception as e:
|
||||
raise Exception(f"Failed to prepare log files for {dnodePath}: {e}")
|
||||
|
||||
tdSql.checkEqual(True, os.path.exists(stubFile101))
|
||||
tdSql.checkEqual(True, os.path.exists(stubGzFile102))
|
||||
tdSql.checkEqual(True, os.path.exists(stubFileNow))
|
||||
tdSql.checkEqual(True, os.path.exists(stubGzFileDel))
|
||||
|
||||
self.openTaosd(f"-c {dnodePath}")
|
||||
self.openTaos(f"-c {dnodePath}")
|
||||
|
||||
tdLog.info("wait %d seconds for log rotate" % (logRotateAfterBoot + 2))
|
||||
time.sleep(logRotateAfterBoot + 2)
|
||||
|
||||
tdSql.checkEqual(True, os.path.exists(stubFile99))
|
||||
tdSql.checkEqual(False, os.path.exists(stubFile101))
|
||||
tdSql.checkEqual(False, os.path.exists(f'{stubFile101}.gz'))
|
||||
tdSql.checkEqual(True, os.path.exists(stubGzFile98))
|
||||
tdSql.checkEqual(True, os.path.exists(f'{stubFileNow}.gz'))
|
||||
tdSql.checkEqual(True, os.path.exists(stubGzFileNow))
|
||||
tdSql.checkEqual(True, os.path.exists(stubGzFileKeep))
|
||||
tdSql.checkEqual(False, os.path.exists(stubGzFile102))
|
||||
tdSql.checkEqual(False, os.path.exists(stubGzFileDel))
|
||||
tdSql.checkEqual(True, os.path.exists(f"{dnodePath}/log/taosdlog.0"))
|
||||
tdSql.checkEqual(True, os.path.exists(f"{dnodePath}/log/taoslog0.0"))
|
||||
|
||||
def run(self):
|
||||
self.checkLogOutput()
|
||||
self.checkLogRotate()
|
||||
self.closeTaosd()
|
||||
self.closeTaos()
|
||||
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
|
|
|
|||
|
|
@ -49,7 +49,12 @@
|
|||
#define SHELL_PKT_LEN "Packet length used for net test, default is 1024 bytes."
|
||||
#define SHELL_PKT_NUM "Packet numbers used for net test, default is 100."
|
||||
#define SHELL_BI_MODE "Set BI mode"
|
||||
#define SHELL_VERSION "Print program version."
|
||||
#define SHELL_LOG_OUTPUT \
|
||||
"Specify log output. Options:\n\r\t\t\t stdout, stderr, /dev/null, <directory>, <directory>/<filename>, " \
|
||||
"<filename>\n\r\t\t\t * If OUTPUT contains an absolute directory, logs will be stored in that directory " \
|
||||
"instead of logDir.\n\r\t\t\t * If OUTPUT contains a relative directory, logs will be stored in the directory " \
|
||||
"combined with logDir and the relative directory."
|
||||
#define SHELL_VERSION "Print program version."
|
||||
|
||||
#ifdef WEBSOCKET
|
||||
#define SHELL_DSN "Use dsn to connect to the cloud server or to a remote server which provides WebSocket connection."
|
||||
|
|
@ -74,6 +79,9 @@ void shellPrintHelp() {
|
|||
printf("%s%s%s%s\r\n", indent, "-l,", indent, SHELL_PKT_LEN);
|
||||
printf("%s%s%s%s\r\n", indent, "-n,", indent, SHELL_NET_ROLE);
|
||||
printf("%s%s%s%s\r\n", indent, "-N,", indent, SHELL_PKT_NUM);
|
||||
#if defined(LINUX)
|
||||
printf("%s%s%s%s\r\n", indent, "-o,", indent, SHELL_LOG_OUTPUT);
|
||||
#endif
|
||||
printf("%s%s%s%s\r\n", indent, "-p,", indent, SHELL_PASSWORD);
|
||||
printf("%s%s%s%s\r\n", indent, "-P,", indent, SHELL_PORT);
|
||||
printf("%s%s%s%s\r\n", indent, "-r,", indent, SHELL_RAW_TIME);
|
||||
|
|
@ -134,6 +142,9 @@ static struct argp_option shellOptions[] = {
|
|||
#endif
|
||||
{"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM},
|
||||
{"bimode", 'B', 0, 0, SHELL_BI_MODE},
|
||||
#if defined(LINUX)
|
||||
{"log-output", 'o', "OUTPUT", 0, SHELL_LOG_OUTPUT},
|
||||
#endif
|
||||
{0},
|
||||
};
|
||||
|
||||
|
|
@ -222,6 +233,23 @@ static int32_t shellParseSingleOpt(int32_t key, char *arg) {
|
|||
case 'N':
|
||||
pArgs->pktNum = atoi(arg);
|
||||
break;
|
||||
#if defined(LINUX)
|
||||
case 'o':
|
||||
if (strlen(arg) >= PATH_MAX) {
|
||||
printf("failed to set log output since length overflow, max length is %d\n", PATH_MAX);
|
||||
return TSDB_CODE_INVALID_CFG;
|
||||
}
|
||||
tsLogOutput = taosMemoryMalloc(PATH_MAX);
|
||||
if (!tsLogOutput) {
|
||||
printf("failed to set log output: '%s' since %s\n", arg, tstrerror(terrno));
|
||||
return terrno;
|
||||
}
|
||||
if (taosExpandDir(arg, tsLogOutput, PATH_MAX) != 0) {
|
||||
printf("failed to expand log output: '%s' since %s\n", arg, tstrerror(terrno));
|
||||
return terrno;
|
||||
}
|
||||
break;
|
||||
#endif
|
||||
#ifdef WEBSOCKET
|
||||
case 'R':
|
||||
pArgs->restful = true;
|
||||
|
|
|
|||
Loading…
Reference in New Issue