diff --git a/docs/en/14-reference/09-error-code.md b/docs/en/14-reference/09-error-code.md index 88b29db217..46f8687843 100644 --- a/docs/en/14-reference/09-error-code.md +++ b/docs/en/14-reference/09-error-code.md @@ -29,6 +29,7 @@ This document details the server error codes that may be encountered when using | 0x8000002A | RPC state already dropped | 1. Engine error, can be ignored, this error code will not be returned to the user side | If returned to the user side, the engine side needs to investigate the issue | | 0x8000002B | RPC msg exceed limit | 1. Single RPC message exceeds the limit, this error code will not be returned to the user side | If returned to the user side, the engine side needs to investigate the issue | + ## common | Error Code | Error Description | Possible Error Scenarios or Reasons | Recommended User Actions | diff --git a/include/common/cos_cp.h b/include/common/cos_cp.h index 29532c0265..e54941f5d3 100644 --- a/include/common/cos_cp.h +++ b/include/common/cos_cp.h @@ -44,7 +44,7 @@ typedef struct { char file_path[TSDB_FILENAME_LEN]; // local file path int64_t file_size; // local file size, for upload - int32_t file_last_modified; // local file last modified time, for upload + int64_t file_last_modified; // local file last modified time, for upload char file_md5[64]; // md5 of the local file content, for upload, reserved char object_name[128]; // object name @@ -67,9 +67,9 @@ int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint); int32_t cos_cp_dump(SCheckpoint* checkpoint); void cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPart* parts, int64_t* consume_bytes); void cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag, uint64_t crc64); -void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int32_t mtime, +void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int64_t mtime, char const* upload_id, int64_t part_size); -bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int32_t mtime); +bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int64_t mtime); void cos_cp_build_download(SCheckpoint* checkpoint, char const* filepath, char const* object_name, int64_t object_size, char const* object_lmtime, char const* object_etag, int64_t part_size); diff --git a/include/os/osFile.h b/include/os/osFile.h index 536dee268a..1c397f3042 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -82,9 +82,9 @@ int32_t taosUnLockFile(TdFilePtr pFile); int32_t taosUmaskFile(int32_t maskVal); -int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *atime); +int32_t taosStatFile(const char *path, int64_t *size, int64_t *mtime, int64_t *atime); int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno); -int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime); +int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int64_t *mtime); bool taosCheckExistFile(const char *pathname); int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 42d4ccf535..7ce4a585eb 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -196,21 +196,21 @@ int32_t taosGetErrSize(); #define TSDB_CODE_TSC_INVALID_JSON TAOS_DEF_ERROR_CODE(0, 0x0221) #define TSDB_CODE_TSC_INVALID_JSON_TYPE TAOS_DEF_ERROR_CODE(0, 0x0222) #define TSDB_CODE_TSC_VALUE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0224) -#define TSDB_CODE_TSC_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0X0229) -#define TSDB_CODE_TSC_STMT_API_ERROR TAOS_DEF_ERROR_CODE(0, 0X022A) -#define TSDB_CODE_TSC_STMT_TBNAME_ERROR TAOS_DEF_ERROR_CODE(0, 0X022B) -#define TSDB_CODE_TSC_STMT_CLAUSE_ERROR TAOS_DEF_ERROR_CODE(0, 0X022C) -#define TSDB_CODE_TSC_QUERY_KILLED TAOS_DEF_ERROR_CODE(0, 0X022D) -#define TSDB_CODE_TSC_NO_EXEC_NODE TAOS_DEF_ERROR_CODE(0, 0X022E) -#define TSDB_CODE_TSC_NOT_STABLE_ERROR TAOS_DEF_ERROR_CODE(0, 0X022F) -#define TSDB_CODE_TSC_STMT_CACHE_ERROR TAOS_DEF_ERROR_CODE(0, 0X0230) -#define TSDB_CODE_TSC_ENCODE_PARAM_ERROR TAOS_DEF_ERROR_CODE(0, 0X0231) -#define TSDB_CODE_TSC_ENCODE_PARAM_NULL TAOS_DEF_ERROR_CODE(0, 0X0232) -#define TSDB_CODE_TSC_COMPRESS_PARAM_ERROR TAOS_DEF_ERROR_CODE(0, 0X0233) -#define TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR TAOS_DEF_ERROR_CODE(0, 0X0234) -#define TSDB_CODE_TSC_FAIL_GENERATE_JSON TAOS_DEF_ERROR_CODE(0, 0X0235) -#define TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR TAOS_DEF_ERROR_CODE(0, 0X0236) -#define TSDB_CODE_TSC_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0X02FF) +#define TSDB_CODE_TSC_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x0229) +#define TSDB_CODE_TSC_STMT_API_ERROR TAOS_DEF_ERROR_CODE(0, 0x022A) +#define TSDB_CODE_TSC_STMT_TBNAME_ERROR TAOS_DEF_ERROR_CODE(0, 0x022B) +#define TSDB_CODE_TSC_STMT_CLAUSE_ERROR TAOS_DEF_ERROR_CODE(0, 0x022C) +#define TSDB_CODE_TSC_QUERY_KILLED TAOS_DEF_ERROR_CODE(0, 0x022D) +#define TSDB_CODE_TSC_NO_EXEC_NODE TAOS_DEF_ERROR_CODE(0, 0x022E) +#define TSDB_CODE_TSC_NOT_STABLE_ERROR TAOS_DEF_ERROR_CODE(0, 0x022F) +#define TSDB_CODE_TSC_STMT_CACHE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0230) +#define TSDB_CODE_TSC_ENCODE_PARAM_ERROR TAOS_DEF_ERROR_CODE(0, 0x0231) +#define TSDB_CODE_TSC_ENCODE_PARAM_NULL TAOS_DEF_ERROR_CODE(0, 0x0232) +#define TSDB_CODE_TSC_COMPRESS_PARAM_ERROR TAOS_DEF_ERROR_CODE(0, 0x0233) +#define TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR TAOS_DEF_ERROR_CODE(0, 0x0234) +#define TSDB_CODE_TSC_FAIL_GENERATE_JSON TAOS_DEF_ERROR_CODE(0, 0x0235) +#define TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR TAOS_DEF_ERROR_CODE(0, 0x0236) +#define TSDB_CODE_TSC_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x02FF) // mnode-common #define TSDB_CODE_MND_REQ_REJECTED TAOS_DEF_ERROR_CODE(0, 0x0300) diff --git a/include/util/tlog.h b/include/util/tlog.h index a6c87593d1..78ca6ffe15 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -41,6 +41,7 @@ extern bool tsAsyncLog; extern bool tsAssert; extern int32_t tsNumOfLogLines; extern int32_t tsLogKeepDays; +extern char *tsLogOutput; extern LogFp tsLogFp; extern int64_t tsNumOfErrorLogs; extern int64_t tsNumOfInfoLogs; @@ -71,7 +72,7 @@ extern int32_t sndDebugFlag; extern int32_t simDebugFlag; extern int32_t tqClientDebugFlag; - +int32_t taosInitLogOutput(const char **ppLogName); int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc); void taosCloseLog(); void taosResetLog(); diff --git a/include/util/tutil.h b/include/util/tutil.h index aa3b774e84..c1b6b2505b 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -56,6 +56,8 @@ void taosIpPort2String(uint32_t ip, uint16_t port, char *str); void *tmemmem(const char *haystack, int hlen, const char *needle, int nlen); int32_t parseCfgReal(const char *str, float *out); +bool tIsValidFileName(const char *fileName, const char *pattern); +bool tIsValidFilePath(const char *filePath, const char *pattern); static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) { T_MD5_CTX context; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index b0be3a4d3b..072cbee2f4 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -41,6 +41,10 @@ #include "cus_name.h" #endif +#ifndef CUS_PROMPT +#define CUS_PROMPT "tao" +#endif + #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -954,14 +958,10 @@ void taos_init_imp(void) { taosHashSetFreeFp(appInfo.pInstMap, destroyAppInst); deltaToUtcInitOnce(); - char logDirName[64] = {0}; -#ifdef CUS_PROMPT - snprintf(logDirName, 64, "%slog", CUS_PROMPT); -#else - (void)snprintf(logDirName, 64, "taoslog"); -#endif - if (taosCreateLog(logDirName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { - (void)printf(" WARING: Create %s failed:%s. configDir=%s\n", logDirName, strerror(errno), configDir); + const char *logName = CUS_PROMPT "slog"; + ENV_ERR_RET(taosInitLogOutput(&logName), "failed to init log output"); + if (taosCreateLog(logName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { + (void)printf(" WARING: Create %s failed:%s. configDir=%s\n", logName, strerror(errno), configDir); tscInitRes = -1; return; } diff --git a/source/common/src/cos.c b/source/common/src/cos.c index a7e69ddc4c..de4f65f2ad 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -775,7 +775,7 @@ _exit: TAOS_RETURN(code); } -static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const char *file, int32_t lmtime, +static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const char *file, int64_t lmtime, char const *object_name, int64_t contentLength, S3PutProperties *put_prop, put_object_callback_data *data) { int32_t code = 0, lino = 0; @@ -963,7 +963,7 @@ _exit: int32_t s3PutObjectFromFile2ByEp(const char *file, const char *object_name, int8_t withcp, int8_t epIndex) { int32_t code = 0; - int32_t lmtime = 0; + int64_t lmtime = 0; const char *filename = 0; uint64_t contentLength = 0; const char *cacheControl = 0, *contentType = 0, *md5 = 0; @@ -1040,7 +1040,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t w static int32_t s3PutObjectFromFileOffsetByEp(const char *file, const char *object_name, int64_t offset, int64_t size, int8_t epIndex) { int32_t code = 0; - int32_t lmtime = 0; + int64_t lmtime = 0; const char *filename = 0; uint64_t contentLength = 0; const char *cacheControl = 0, *contentType = 0, *md5 = 0; @@ -1847,7 +1847,7 @@ _exit: typedef struct { int64_t size; - int32_t atime; + int64_t atime; char name[TSDB_FILENAME_LEN]; } SEvictFile; diff --git a/source/common/src/cos_cp.c b/source/common/src/cos_cp.c index 078b14c9e8..3469e8ecec 100644 --- a/source/common/src/cos_cp.c +++ b/source/common/src/cos_cp.c @@ -350,7 +350,7 @@ void cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag checkpoint->parts[part_index].crc64 = crc64; } -void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int32_t mtime, +void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int64_t mtime, char const* upload_id, int64_t part_size) { int i = 0; @@ -375,7 +375,7 @@ void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t static bool cos_cp_verify_md5(SCheckpoint* cp) { return true; } -bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int32_t mtime) { +bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int64_t mtime) { if (cos_cp_verify_md5(checkpoint) && checkpoint->file_size == size && checkpoint->file_last_modified == mtime) { return true; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 46026b586b..5076006f29 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -17,6 +17,7 @@ #include "tglobal.h" #include "defines.h" #include "os.h" +#include "osString.h" #include "tconfig.h" #include "tgrant.h" #include "tlog.h" @@ -1010,12 +1011,43 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { TAOS_RETURN(TSDB_CODE_SUCCESS); } +static int32_t taosSetLogOutput(SConfig *pCfg) { + if (tsLogOutput) { + char *pLog = tsLogOutput; + char *pEnd = NULL; + if (strcasecmp(pLog, "stdout") && strcasecmp(pLog, "stderr") && strcasecmp(pLog, "/dev/null")) { + if ((pEnd = strrchr(pLog, '/')) || (pEnd = strrchr(pLog, '\\'))) { + int32_t pathLen = POINTER_DISTANCE(pEnd, pLog) + 1; + if (*pLog == '/' || *pLog == '\\') { + if (pathLen <= 0 || pathLen > PATH_MAX) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + tstrncpy(tsLogDir, pLog, pathLen); + } else { + int32_t len = strlen(tsLogDir); + if (len < 0 || len >= (PATH_MAX - 1)) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + if (len == 0 || (tsLogDir[len - 1] != '/' && tsLogDir[len - 1] != '\\')) { + tsLogDir[len++] = TD_DIRSEP_CHAR; + } + int32_t remain = PATH_MAX - len - 1; + if (remain < pathLen) TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + tstrncpy(tsLogDir + len, pLog, pathLen); + } + TAOS_CHECK_RETURN(cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true)); + } + } else { + tstrncpy(tsLogDir, pLog, PATH_MAX); + TAOS_CHECK_RETURN(cfgSetItem(pCfg, "logDir", tsLogDir, CFG_STYPE_DEFAULT, true)); + } + } + return 0; +} + static int32_t taosSetClientLogCfg(SConfig *pCfg) { SConfigItem *pItem = NULL; TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "logDir"); tstrncpy(tsLogDir, pItem->str, PATH_MAX); TAOS_CHECK_RETURN(taosExpandDir(tsLogDir, tsLogDir, PATH_MAX)); + TAOS_CHECK_RETURN(taosSetLogOutput(pCfg)); TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "minimalLogDirGB"); tsLogSpace.reserved = (int64_t)(((double)pItem->fval) * 1024 * 1024 * 1024); @@ -1860,8 +1892,10 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile } if (tsc) { + TAOS_CHECK_GOTO(taosSetClientLogCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit); } else { + TAOS_CHECK_GOTO(taosSetClientLogCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosUpdateServerCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosSetServerCfg(tsCfg), &lino, _exit); diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index ade5e16894..4aa76d9284 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -49,6 +49,7 @@ #define DM_ENV_CMD "The env cmd variable string to use when configuring the server, such as: -e 'TAOS_FQDN=td1'." #define DM_ENV_FILE "The env variable file path to use when configuring the server, default is './.env', .env text can be 'TAOS_FQDN=td1'." #define DM_MACHINE_CODE "Get machine code." +#define DM_LOG_OUTPUT "Specify log output. Options:\n\r\t\t\t stdout, stderr, /dev/null, , /, \n\r\t\t\t * If OUTPUT contains an absolute directory, logs will be stored in that directory instead of logDir.\n\r\t\t\t * If OUTPUT contains a relative directory, logs will be stored in the directory combined with logDir and the relative directory." #define DM_VERSION "Print program version." #define DM_EMAIL "" #define DM_MEM_DBG "Enable memory debug" @@ -239,6 +240,32 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { } } else if (strcmp(argv[i], "-k") == 0) { global.generateGrant = true; +#if defined(LINUX) + } else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--log-output") == 0 || + strncmp(argv[i], "--log-output=", 13) == 0) { + if ((i < argc - 1) || ((i == argc - 1) && strncmp(argv[i], "--log-output=", 13) == 0)) { + int32_t klen = strlen(argv[i]); + int32_t vlen = klen < 13 ? strlen(argv[++i]) : klen - 13; + const char *val = argv[i]; + if (klen >= 13) val += 13; + if (vlen <= 0 || vlen >= PATH_MAX) { + printf("failed to set log output since invalid vlen:%d, valid range: [1, %d)\n", vlen, PATH_MAX); + return TSDB_CODE_INVALID_CFG; + } + tsLogOutput = taosMemoryMalloc(PATH_MAX); + if (!tsLogOutput) { + printf("failed to set log output: '%s' since %s\n", val, tstrerror(terrno)); + return terrno; + } + if (taosExpandDir(val, tsLogOutput, PATH_MAX) != 0) { + printf("failed to expand log output: '%s' since %s\n", val, tstrerror(terrno)); + return terrno; + } + } else { + printf("'%s' requires a parameter\n", argv[i]); + return TSDB_CODE_INVALID_CFG; + } +#endif } else if (strcmp(argv[i], "-y") == 0) { global.generateCode = true; if (i < argc - 1) { @@ -316,6 +343,9 @@ static void dmPrintHelp() { printf("%s%s%s%s\n", indent, "-e,", indent, DM_ENV_CMD); printf("%s%s%s%s\n", indent, "-E,", indent, DM_ENV_FILE); printf("%s%s%s%s\n", indent, "-k,", indent, DM_MACHINE_CODE); +#if defined(LINUX) + printf("%s%s%s%s\n", indent, "-o, --log-output=OUTPUT", indent, DM_LOG_OUTPUT); +#endif printf("%s%s%s%s\n", indent, "-y,", indent, DM_SET_ENCRYPTKEY); printf("%s%s%s%s\n", indent, "-dm,", indent, DM_MEM_DBG); printf("%s%s%s%s\n", indent, "-V,", indent, DM_VERSION); @@ -340,8 +370,11 @@ static int32_t dmCheckS3() { } static int32_t dmInitLog() { - return taosCreateLog(CUS_PROMPT "dlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, - 0); + const char *logName = CUS_PROMPT "dlog"; + + TAOS_CHECK_RETURN(taosInitLogOutput(&logName)); + + return taosCreateLog(logName, 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0); } static void taosCleanupArgs() { diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 74ec310f70..7859ee4c66 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -660,7 +660,7 @@ static int32_t tsdbDoS3Migrate(SRTNer *rtner) { int32_t lcn = fobj->f->lcn; if (/*lcn < 1 && */ taosCheckExistFile(fobj->fname)) { - int32_t mtime = 0; + int64_t mtime = 0; int64_t size = 0; int32_t r = taosStatFile(fobj->fname, &size, &mtime, NULL); if (size > chunksize && mtime < rtner->now - tsS3UploadDelaySec) { @@ -687,7 +687,7 @@ static int32_t tsdbDoS3Migrate(SRTNer *rtner) { tsdbTFileLastChunkName(rtner->tsdb, fobj->f, fname1); if (taosCheckExistFile(fname1)) { - int32_t mtime = 0; + int64_t mtime = 0; int64_t size = 0; if (taosStatFile(fname1, &size, &mtime, NULL) != 0) { tsdbError("vgId:%d, %s failed at %s:%d ", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, __LINE__); diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 0ee14f7820..bbfd43d5f7 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -1564,8 +1564,8 @@ static void udfdPrintVersion() { } static int32_t udfdInitLog() { - char logName[12] = {0}; - snprintf(logName, sizeof(logName), "%slog", "udfd"); + const char *logName = "udfdlog"; + TAOS_CHECK_RETURN(taosInitLogOutput(&logName)); return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, NULL, 0); } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 06e3765fcf..0fe074084f 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1724,9 +1724,8 @@ void syncNodeResetElectTimer(SSyncNode* pSyncNode) { electMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine); } - // TODO check return value if ((code = syncNodeRestartElectTimer(pSyncNode, electMS)) != 0) { - sError("vgId:%d, failed to restart elect timer since %s", pSyncNode->vgId, tstrerror(code)); + sWarn("vgId:%d, failed to restart elect timer since %s", pSyncNode->vgId, tstrerror(code)); return; }; diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index efb71b5714..af6aab5d2b 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -1152,11 +1152,11 @@ int32_t syncLogReplProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncApp int32_t code = 0; if (pMgr->restored) { if ((code = syncLogReplContinue(pMgr, pNode, pMsg)) != 0) { - sError("vgId:%d, failed to continue sync log repl since %s", pNode->vgId, tstrerror(code)); + sWarn("vgId:%d, failed to continue sync log repl since %s", pNode->vgId, tstrerror(code)); } } else { if ((code = syncLogReplRecover(pMgr, pNode, pMsg)) != 0) { - sError("vgId:%d, failed to recover sync log repl since %s", pNode->vgId, tstrerror(code)); + sWarn("vgId:%d, failed to recover sync log repl since %s", pNode->vgId, tstrerror(code)); } } (void)taosThreadMutexUnlock(&pBuf->mutex); diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 7466aaf66e..7b5b40f327 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -75,8 +75,10 @@ int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode) { continue; } SSyncLogReplMgr* pMgr = pNode->logReplMgrs[i]; - if (syncLogReplStart(pMgr, pNode) != 0) { - sError("vgId:%d, failed to start log replication to dnode:%d", pNode->vgId, DID(&(pNode->replicasId[i]))); + int32_t ret = 0; + if ((ret = syncLogReplStart(pMgr, pNode)) != 0) { + sWarn("vgId:%d, failed to start log replication to dnode:%d since %s", pNode->vgId, DID(&(pNode->replicasId[i])), + tstrerror(ret)); } } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index b40a9eeefe..78f13a58ab 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -326,7 +326,7 @@ static int32_t walRepairLogFileTs(SWal* pWal, bool* updateMeta) { } walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); - int32_t mtime = 0; + int64_t mtime = 0; if (taosStatFile(fnameStr, NULL, &mtime, NULL) < 0) { wError("vgId:%d, failed to stat file due to %s, file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 5db60df1ec..25245d2df9 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -303,7 +303,7 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue; char filename[1024]; - (void)snprintf(filename, sizeof(filename), "%s/%s", dirname, taosGetDirEntryName(de)); + (void)snprintf(filename, sizeof(filename), "%s%s%s", dirname, TD_DIRSEP, taosGetDirEntryName(de)); if (taosDirEntryIsDir(de)) { continue; } else { @@ -349,7 +349,7 @@ int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) { wordfree(&full_path); // FALL THROUGH default: - return code; + return terrno = TSDB_CODE_INVALID_PARA; } if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) { diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 41b34a9030..832deb574d 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -89,7 +89,7 @@ int32_t osDefaultInit() { } tstrncpy(tsDataDir, TD_DATA_DIR_PATH, sizeof(tsDataDir)); tstrncpy(tsLogDir, TD_LOG_DIR_PATH, sizeof(tsLogDir)); - if (strlen(tsTempDir) == 0){ + if (strlen(tsTempDir) == 0) { tstrncpy(tsTempDir, TD_TMP_DIR_PATH, sizeof(tsTempDir)); } diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index a3747535f6..5030eb1f67 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -273,7 +273,7 @@ int32_t taosRenameFile(const char *oldName, const char *newName) { #endif } -int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *atime) { +int32_t taosStatFile(const char *path, int64_t *size, int64_t *mtime, int64_t *atime) { OS_PARAM_CHECK(path); #ifdef WINDOWS struct _stati64 fileStat; @@ -544,7 +544,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { return liOffset.QuadPart; } -int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { +int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int64_t *mtime) { if (pFile == NULL || pFile->hFile == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; @@ -571,7 +571,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { ULARGE_INTEGER ull; ull.LowPart = lastWriteTime.dwLowDateTime; ull.HighPart = lastWriteTime.dwHighDateTime; - *mtime = (int32_t)((ull.QuadPart - 116444736000000000ULL) / 10000000ULL); + *mtime = (int64_t)((ull.QuadPart - 116444736000000000ULL) / 10000000ULL); } return 0; } @@ -937,7 +937,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { return ret; } -int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { +int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int64_t *mtime) { if (pFile == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index f70b145dbc..693935575a 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -51,6 +51,13 @@ #define LOG_EDITION_FLG ("C") #endif +typedef enum { + LOG_OUTPUT_FILE = 0, // default + LOG_OUTPUT_STDOUT = 1, // stdout set by -o option on the command line + LOG_OUTPUT_STDERR = 2, // stderr set by -o option on the command line + LOG_OUTPUT_NULL = 4, // /dev/null set by -o option on the command line +} ELogOutputType; + typedef struct { char *buffer; int32_t buffStart; @@ -73,6 +80,7 @@ typedef struct { int32_t openInProgress; int64_t lastKeepFileSec; int64_t timestampToday; + int8_t outputType; // ELogOutputType pid_t pid; char logName[PATH_MAX]; char slowLogName[PATH_MAX]; @@ -86,6 +94,7 @@ static int8_t tsLogInited = 0; static SLogObj tsLogObj = {.fileNum = 1, .slowHandle = NULL}; static int64_t tsAsyncLogLostLines = 0; static int32_t tsDaylightActive; /* Currently in daylight saving time. */ +static SRWLatch tsLogRotateLatch = 0; bool tsLogEmbedded = 0; bool tsAsyncLog = true; @@ -96,6 +105,7 @@ bool tsAssert = true; #endif int32_t tsNumOfLogLines = 10000000; int32_t tsLogKeepDays = 0; +char *tsLogOutput = NULL; LogFp tsLogFp = NULL; int64_t tsNumOfErrorLogs = 0; int64_t tsNumOfInfoLogs = 0; @@ -234,6 +244,58 @@ int32_t taosInitSlowLog() { return 0; } +int32_t taosInitLogOutput(const char **ppLogName) { + const char *pLog = tsLogOutput; + const char *pLogName = NULL; + if (pLog) { + if (!tIsValidFilePath(pLog, NULL)) { + fprintf(stderr, "invalid log output destination:%s, contains illegal char\n", pLog); + return TSDB_CODE_INVALID_CFG; + } + if (0 == strcasecmp(pLog, "stdout")) { + tsLogObj.outputType = LOG_OUTPUT_STDOUT; + if (ppLogName) *ppLogName = pLog; + return 0; + } + if (0 == strcasecmp(pLog, "stderr")) { + tsLogObj.outputType = LOG_OUTPUT_STDERR; + if (ppLogName) *ppLogName = pLog; + return 0; + } + if (0 == strcasecmp(pLog, "/dev/null")) { + tsLogObj.outputType = LOG_OUTPUT_NULL; + if (ppLogName) *ppLogName = pLog; + return 0; + } + int32_t len = strlen(pLog); + if (len < 1) { + fprintf(stderr, "invalid log output destination:%s, should not be empty\n", pLog); + return TSDB_CODE_INVALID_CFG; + } + const char *p = pLog + (len - 1); + if (*p == '/' || *p == '\\') { + return 0; + } + + if ((p = strrchr(pLog, '/')) || (p = strrchr(pLog, '\\'))) { + pLogName = p + 1; + } else { + pLogName = pLog; + } + if (strcmp(pLogName, ".") == 0 || strcmp(pLogName, "..") == 0) { + fprintf(stderr, "invalid log output destination:%s\n", pLog); + return TSDB_CODE_INVALID_CFG; + } + + if (!tIsValidFileName(pLogName, NULL)) { + fprintf(stderr, "invalid log output destination:%s, contains illegal char\n", pLog); + return TSDB_CODE_INVALID_CFG; + } + if (ppLogName) *ppLogName = pLogName; + } + return 0; +} + int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) { if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0; int32_t code = osUpdate(); @@ -241,6 +303,11 @@ int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) { uError("failed to update os info, reason:%s", tstrerror(code)); } + if (tsLogObj.outputType == LOG_OUTPUT_STDOUT || tsLogObj.outputType == LOG_OUTPUT_STDERR || + tsLogObj.outputType == LOG_OUTPUT_NULL) { + return 0; + } + TAOS_CHECK_RETURN(taosInitNormalLog(logName, maxFiles)); if (tsc) { TAOS_CHECK_RETURN(taosInitSlowLog()); @@ -283,6 +350,7 @@ void taosCloseLog() { taosMemoryFreeClear(tsLogObj.logHandle); tsLogObj.logHandle = NULL; } + taosMemoryFreeClear(tsLogOutput); } static bool taosLockLogFile(TdFilePtr pFile) { @@ -340,10 +408,6 @@ static void taosKeepOldLog(char *oldName) { } } } - - if (tsLogKeepDays > 0) { - taosRemoveOldFiles(tsLogDir, tsLogKeepDays); - } } typedef struct { TdFilePtr pOldFile; @@ -392,11 +456,16 @@ static OldFileKeeper *taosOpenNewFile() { static void *taosThreadToCloseOldFile(void *param) { if (!param) return NULL; + taosWLockLatch(&tsLogRotateLatch); OldFileKeeper *oldFileKeeper = (OldFileKeeper *)param; taosSsleep(20); taosCloseLogByFd(oldFileKeeper->pOldFile); taosKeepOldLog(oldFileKeeper->keepName); taosMemoryFree(oldFileKeeper); + if (tsLogKeepDays > 0) { + taosRemoveOldFiles(tsLogDir, tsLogKeepDays); + } + taosWUnLockLatch(&tsLogRotateLatch); return NULL; } @@ -532,8 +601,8 @@ static void decideLogFileName(const char *fn, int32_t maxFileNum) { static void decideLogFileNameFlag() { char name[PATH_MAX + 50] = "\0"; - int32_t logstat0_mtime = 0; - int32_t logstat1_mtime = 0; + int64_t logstat0_mtime = 0; + int64_t logstat1_mtime = 0; bool log0Exist = false; bool log1Exist = false; @@ -680,10 +749,19 @@ static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *b } } - if (dflag & DEBUG_SCREEN) { + int fd = 0; + if (tsLogObj.outputType == LOG_OUTPUT_FILE) { + if (dflag & DEBUG_SCREEN) fd = 1; + } else if (tsLogObj.outputType == LOG_OUTPUT_STDOUT) { + fd = 1; + } else if (tsLogObj.outputType == LOG_OUTPUT_STDERR) { + fd = 2; + } + + if (fd) { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-result" - if (write(1, buffer, (uint32_t)len) < 0) { + if (write(fd, buffer, (uint32_t)len) < 0) { TAOS_UNUSED(printf("failed to write log to screen, reason:%s\n", strerror(errno))); } #pragma GCC diagnostic pop @@ -999,6 +1077,92 @@ static void taosWriteLog(SLogBuff *pLogBuf) { pLogBuf->writeInterval = 0; } +#define LOG_ROTATE_INTERVAL 3600 +#if !defined(TD_ENTERPRISE) || defined(ASSERT_NOT_CORE) +#define LOG_INACTIVE_TIME 7200 +#define LOG_ROTATE_BOOT 900 +#else +#define LOG_INACTIVE_TIME 5 +#define LOG_ROTATE_BOOT (LOG_INACTIVE_TIME + 1) +#endif + +static void *taosLogRotateFunc(void *param) { + setThreadName("logRotate"); + int32_t code = 0; + taosWLockLatch(&tsLogRotateLatch); + // compress or remove the old log files + TdDirPtr pDir = taosOpenDir(tsLogDir); + if (!pDir) goto _exit; + TdDirEntryPtr de = NULL; + while ((de = taosReadDir(pDir))) { + if (taosDirEntryIsDir(de)) { + continue; + } + char *fname = taosGetDirEntryName(de); + if (!fname) { + continue; + } + + char *pSec = strrchr(fname, '.'); + if (!pSec) { + continue; + } + char *pIter = pSec; + bool isSec = true; + while (*(++pIter)) { + if (!isdigit(*pIter)) { + isSec = false; + break; + } + } + if (!isSec) { + continue; + } + + int64_t fileSec = 0; + if ((code = taosStr2int64(pSec + 1, &fileSec)) != 0) { + uWarn("%s:%d failed to convert %s to int64 since %s", __func__, __LINE__, pSec + 1, tstrerror(code)); + continue; + } + if (fileSec <= 100) { + continue; + } + + char fullName[PATH_MAX] = {0}; + snprintf(fullName, sizeof(fullName), "%s%s%s", tsLogDir, TD_DIRSEP, fname); + + int64_t mtime = 0; + if ((code = taosStatFile(fullName, NULL, &mtime, NULL)) != 0) { + uWarn("%s:%d failed to stat file %s since %s", __func__, __LINE__, fullName, tstrerror(code)); + continue; + } + + int64_t inactiveSec = taosGetTimestampMs() / 1000 - mtime; + + if (inactiveSec < LOG_INACTIVE_TIME) { + continue; + } + + int32_t days = inactiveSec / 86400 + 1; + if (tsLogKeepDays > 0 && days > tsLogKeepDays) { + TAOS_UNUSED(taosRemoveFile(fullName)); + uInfo("file:%s is removed, days:%d, keepDays:%d, sed:%" PRId64, fullName, days, tsLogKeepDays, fileSec); + } else { + taosKeepOldLog(fullName); // compress + } + } + if ((code = taosCloseDir(&pDir)) != 0) { + uWarn("%s:%d failed to close dir %s since %s\n", __func__, __LINE__, tsLogDir, tstrerror(code)); + } + + if (tsLogKeepDays > 0) { + taosRemoveOldFiles(tsLogDir, tsLogKeepDays); + } +_exit: + taosWUnLockLatch(&tsLogRotateLatch); + return NULL; +} + static void *taosAsyncOutputLog(void *param) { SLogBuff *pLogBuf = (SLogBuff *)tsLogObj.logHandle; SLogBuff *pSlowBuf = (SLogBuff *)tsLogObj.slowHandle; @@ -1007,6 +1171,7 @@ static void *taosAsyncOutputLog(void *param) { int32_t count = 0; int32_t updateCron = 0; int32_t writeInterval = 0; + int64_t lastCheckSec = taosGetTimestampMs() / 1000 - (LOG_ROTATE_INTERVAL - LOG_ROTATE_BOOT); while (1) { if (pSlowBuf) { @@ -1032,6 +1197,26 @@ static void *taosAsyncOutputLog(void *param) { if (pSlowBuf) taosWriteSlowLog(pSlowBuf); break; } + + // process the log rotation every LOG_ROTATE_INTERVAL + int64_t curSec = taosGetTimestampMs() / 1000; + if (curSec >= lastCheckSec) { + if ((curSec - lastCheckSec) >= LOG_ROTATE_INTERVAL) { + TdThread thread; + TdThreadAttr attr; + (void)taosThreadAttrInit(&attr); + (void)taosThreadAttrSetDetachState(&attr, PTHREAD_CREATE_DETACHED); + if (taosThreadCreate(&thread, &attr, taosLogRotateFunc, tsLogObj.logHandle) == 0) { + uInfo("process log rotation"); + lastCheckSec = curSec; + } else { + uWarn("failed to create thread to process log rotation"); + } + (void)taosThreadAttrDestroy(&attr); + } + } else if (curSec < lastCheckSec) { + lastCheckSec = curSec; + } } return NULL; diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index 48338e7996..dd3c6ece40 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "tutil.h" #include "tlog.h" +#include "regex.h" void *tmemmem(const char *haystack, int32_t hlen, const char *needle, int32_t nlen) { const char *limit; @@ -520,3 +521,29 @@ int32_t parseCfgReal(const char *str, float *out) { *out = val; return TSDB_CODE_SUCCESS; } + +bool tIsValidFileName(const char *fileName, const char *pattern) { + const char *fileNamePattern = "^[a-zA-Z0-9_.-]+$"; + + regex_t fileNameReg; + + if (pattern) fileNamePattern = pattern; + + if (regcomp(&fileNameReg, fileNamePattern, REG_EXTENDED) != 0) { + fprintf(stderr, "failed to compile file name pattern:%s\n", fileNamePattern); + return false; + } + + int32_t code = regexec(&fileNameReg, fileName, 0, NULL, 0); + regfree(&fileNameReg); + if (code != 0) { + return false; + } + return true; +} + +bool tIsValidFilePath(const char *filePath, const char *pattern) { + const char *filePathPattern = "^[a-zA-Z0-9:/\\_.-]+$"; + + return tIsValidFileName(filePath, pattern ? pattern : filePathPattern); +} \ No newline at end of file diff --git a/tests/army/sys/checkErrorCode.py b/tests/army/sys/checkErrorCode.py new file mode 100644 index 0000000000..71aa43b02d --- /dev/null +++ b/tests/army/sys/checkErrorCode.py @@ -0,0 +1,181 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import random +import taos + +import frame +import frame.eos +import frame.etime +import frame.etool +from frame.log import * +from frame.sql import * +from frame.cases import * +from frame.caseBase import * +from frame.srvCtl import * +from frame import * + + +class TDTestCase(TBase): + # parse line + def parseLine(self, line): + line = line.strip() + PRE_DEFINE = "#define TSDB_CODE_" + n = len(PRE_DEFINE) + if line[:n] != PRE_DEFINE: + return None + # TAOS_DEF_ERROR_CODE(0, 0x000B) + pos = line.find("TAOS_DEF_ERROR_CODE(0, 0x", n) + if pos == -1: + tdLog.info(f"not found \"TAOS_DEF_ERROR_CODE(0, \" line={line}") + return None + + code = line[pos:].strip() + pos = code.find(")") + if pos == -1: + tdLog.info(f"not found \")\", line={line}") + return None + code = code[:pos] + if len(code) != 4: + tdLog.info(f"code is len not 4 len:{len(code)} subcode={code}\")\", line={line}") + return None + + # return + return "0x8000" + code + + # ignore error + def ignoreCode(self, code): + ignoreCodes = {"0x00008, 0x000009"} + if code in ignoreCodes: + return True + else: + return False + + # read valid code + def readHeadCodes(self, hFile): + codes = [] + start = False + # read + with open(hFile) as file: + for line in file: + code = self.parseLine(line) + # invalid + if code == None: + continue + # ignore + if self.ignoreCode(code): + tdLog.info(f"ignore error {code}\n") + # valid + if code == 0: + start = True + if start: + codes.append(code) + # return + return codes + + # parse doc lines + def parseDocLine(self, line): + line = line.strip() + PRE_DEFINE = "| 0x8000" + n = len(PRE_DEFINE) + if line[:n] != PRE_DEFINE: + return None + line = line[2:] + cols = line.split("|") + # remove blank + cols = [col.strip() for col in cols] + + # return + return cols + + + # read valid code + def readDocCodes(self, docFile): + codes = [] + start = False + # read + with open(docFile) as file: + for line in file: + code = self.parseDocLine(line) + # invalid + if code == None: + continue + # valid + if start: + codes.append(code) + # return + return codes + + # check + def checkConsistency(self, docCodes, codes): + diff = False + # len + docLen = len(docCodes) + len = len(codes) + tdLog.info("head file codes = {len} doc file codes={docLen} \n") + + if docLen > len: + maxLen = docLen + else: + maxLen = len + + for i in range(maxLen): + if i < len and i < docLen: + if codes[i] == docCodes[i][0]: + tdLog.info(f" i={i} same head code: {codes[i]} doc code:{docCodes[i][0]}\n") + else: + tdLog.info(f" i={i} diff head code: {codes[i]} doc code:{docCodes[i][0]}\n") + diff = True + elif i < len: + tdLog.info(f" i={i} diff head code: {codes[i]} doc code: None\n") + diff = True + elif i < docLen: + tdLog.info(f" i={i} diff head code: None doc code: {docCodes[i][0]}\n") + diff = True + + # result + if diff: + tdLog.exit("check error code consistency failed.\n") + + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + + # read head error code + hFile = "../../include/util/taoserror.h" + codes = self.readHeadCodes(hFile) + + # read zh codes + zhDoc = "../../docs/zh/14-reference/09-error-code.md" + zhCodes = self.readDocCodes(zhDoc, codes) + + # read en codes + enDoc = "../../docs/en/14-reference/09-error-code.md" + enCodes = self.readDocCodes(enDoc, codes) + + # check zh + tdLog.info(f"check zh docs ...\n") + self.checkConsistency(zhCodes, codes) + + # check en + tdLog.info(f"check en docs ...\n") + self.checkConsistency(enCodes, codes) + + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/whole/checkErrorCode.py b/tests/army/whole/checkErrorCode.py new file mode 100644 index 0000000000..156cacb482 --- /dev/null +++ b/tests/army/whole/checkErrorCode.py @@ -0,0 +1,241 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import random +import taos + +import frame +import frame.eos +import frame.etime +import frame.etool + +from frame.log import * +from frame.sql import * +from frame.cases import * +from frame.caseBase import * +from frame.srvCtl import * +from frame import * + +ignoreCodes = [ + '0x80000023', '0x80000024', '0x80000025', '0x80000026', '0x80000027', '0x80000028', '0x80000029', '0x8000002A', '0x80000109', '0x80000110', + '0x80000129', '0x8000012C', '0x8000012D', '0x8000012E', '0x8000012F', '0x80000136', '0x80000137', '0x80000138', '0x80000139', '0x8000013A', + '0x8000013B', '0x80000200', '0x80000201', '0x80000202', '0x80000203', '0x80000204', '0x80000205', '0x80000206', '0x8000020B', '0x8000020E', + '0x80000210', '0x80000212', '0x80000214', '0x80000215', '0x80000217', '0x8000021B', '0x8000021C', '0x8000021D', '0x8000021E', '0x80000220', + '0x80000221', '0x8000022C', '0x80000232', '0x80000233', '0x80000234', '0x80000235', '0x80000236', '0x800002FF', '0x80000300', '0x80000316', + '0x80000317', '0x80000338', '0x80000339', '0x8000033F', '0x80000343', '0x80000345', '0x80000356', '0x80000359', '0x8000035A', '0x8000035B', + '0x8000035C', '0x8000035D', '0x80000362', '0x8000038C', '0x8000038E', '0x8000039B', '0x8000039C', '0x8000039D', '0x8000039E', '0x800003A6', + '0x800003A7', '0x800003AA', '0x800003AB', '0x800003AC', '0x800003AD', '0x800003B0', '0x800003B2', '0x800003B4', '0x800003B5', '0x800003BA', + '0x800003C0', '0x800003C1', '0x800003D0', '0x800003D8', '0x800003D9', '0x800003E2', '0x800003F4', '0x800003F8', '0x80000412', '0x80000413', + '0x80000414', '0x80000415', '0x80000416', '0x80000417', '0x80000418', '0x80000419', '0x80000420', '0x80000421', '0x80000422', '0x80000423', + '0x80000424', '0x80000425', '0x80000426', '0x80000427', '0x80000428', '0x80000429', '0x8000042A', '0x80000430', '0x80000431', '0x80000432', + '0x80000433', '0x80000434', '0x80000435', '0x80000436', '0x80000437', '0x80000438', '0x80000440', '0x80000441', '0x80000442', '0x80000443', + '0x80000444', '0x80000445', '0x80000446', '0x80000447', '0x80000485', '0x80000486', '0x800004A0', '0x800004A1', '0x800004B1', '0x800004B2', + '0x800004B3', '0x80000504', '0x80000528', '0x80000532', '0x80000533', '0x80000534', '0x80000535', '0x80000536', '0x80000537', '0x80000538', + '0x80000539', '0x80000601', '0x80000607', '0x8000060A', '0x8000060D', '0x8000060E', '0x8000060F', '0x80000610', '0x80000611', '0x80000612', + '0x80000613', '0x80000614', '0x80000615', '0x80000616', '0x8000061C', '0x8000061E', '0x80000701', '0x80000705', '0x80000706', '0x80000707', + '0x80000708', '0x8000070B', '0x8000070C', '0x8000070E', '0x8000070F', '0x80000712', '0x80000723', '0x80000724', '0x80000725', '0x80000726', + '0x80000727', '0x80000728', '0x8000072A', '0x8000072C', '0x8000072D', '0x8000072E', '0x80000730', '0x80000731', '0x80000732', '0x80000733', + '0x80000734', '0x80000735', '0x80000736', '0x80000737', '0x80000738', '0x8000080E', '0x8000080F', '0x80000810', '0x80000811', '0x80000812', + '0x80000813', '0x80000814', '0x80000815', '0x80000816', '0x80000817', '0x80000818', '0x80000819', '0x80000820', '0x80000821', '0x80000822', + '0x80000823', '0x80000824', '0x80000825', '0x80000826', '0x80000827', '0x80000828', '0x80000829', '0x8000082A', '0x8000082B', '0x8000082C', + '0x8000082D', '0x80000907', '0x80000919', '0x8000091A', '0x8000091B', '0x8000091C', '0x8000091D', '0x8000091E', '0x8000091F', '0x80000A00', + '0x80000A01', '0x80000A03', '0x80000A06', '0x80000A07', '0x80000A08', '0x80000A09', '0x80000A0A', '0x80000A0B', '0x80000A0E', '0x80002206', + '0x80002207', '0x80002406', '0x80002407', '0x80002503', '0x80002506', '0x80002507', '0x8000261B', '0x80002653', '0x80002668', '0x80002669', + '0x8000266A', '0x8000266B', '0x8000266C', '0x8000266D', '0x8000266E', '0x8000266F', '0x80002670', '0x80002671', '0x80002672', '0x80002673', + '0x80002674', '0x80002675', '0x80002676', '0x80002677', '0x80002678', '0x80002679', '0x8000267A', '0x8000267B', '0x8000267C', '0x8000267D', + '0x8000267E', '0x8000267F', '0x80002680', '0x80002681', '0x80002682', '0x80002683', '0x80002684', '0x80002685', '0x80002703', '0x80002806', + '0x80002807', '0x80002808', '0x80002809', '0x8000280A', '0x8000280B', '0x8000280C', '0x8000280D', '0x8000280E', '0x8000280F', '0x80002810', + '0x80002811', '0x80002812', '0x80002813', '0x80002814', '0x80002815', '0x8000290B', '0x80002920', '0x80003003', '0x80003006', '0x80003106', + '0x80003107', '0x80003108', '0x80003109', '0x80003110', '0x80003111', '0x80003112', '0x80003250', '0x80004003', '0x80004004', '0x80004005', + '0x80004006', '0x80004007', '0x80004008', '0x80004009', '0x80004010', '0x80004011', '0x80004012', '0x80004013', '0x80004014', '0x80004015', + '0x80004016', '0x80004102', '0x80004103', '0x80004104', '0x80004105', '0x80004106', '0x80004107', '0x80004108', '0x80004109', '0x80005100', + '0x80005101', '0x80006000', '0x80006100', '0x80006101', '0x80006102', '0x80000019', '0x80002639', '0x80002666'] + + +class TDTestCase(TBase): + # parse line + def parseLine(self, line, show): + line = line.strip() + # PRE_DEFINE + PRE_DEFINE = "#define TSDB_CODE_" + n = len(PRE_DEFINE) + if line[:n] != PRE_DEFINE: + return None + # MID_DEFINE + MID_DEFINE = "TAOS_DEF_ERROR_CODE(0, 0x" + pos = line.find(MID_DEFINE, n) + if pos == -1: + if show: + tdLog.info(f"not found \"{MID_DEFINE}\" line={line}") + return None + start = pos + len(MID_DEFINE) + code = line[start:].strip() + # ) + pos = code.find(")") + if pos == -1: + if show: + tdLog.info(f"not found \")\", code={code}") + return None + # check len + code = code[:pos] + if len(code) != 4: + if show: + tdLog.info(f"code is len not 4 len:{len(code)} subcode={code}\")\", line={line}") + return None + + # return + return "0x8000" + code + + # ignore error + def ignoreCode(self, code): + if code in ignoreCodes: + return True + else: + return False + + # read valid code + def readHeadCodes(self, hFile): + codes = [] + start = False + # read + with open(hFile) as file: + for line in file: + code = self.parseLine(line, start) + # invalid + if code == None: + continue + #print(code) + # valid + if code == "0x8000000B": + start = True + if start: + codes.append(code) + # return + return codes + + # parse doc lines + def parseDocLine(self, line): + line = line.strip() + PRE_DEFINE = "| 0x8000" + n = len(PRE_DEFINE) + if line[:n] != PRE_DEFINE: + return None + line = line[2:] + cols = line.split("|") + # remove blank + cols = [col.strip() for col in cols] + #print(cols) + + # return + return cols + + + # read valid code + def readDocCodes(self, docFile): + codes = [] + start = False + # read + with open(docFile) as file: + for line in file: + code = self.parseDocLine(line) + # invalid + if code == None: + continue + # valid + codes.append(code) + + # return + return codes + + # check + def checkConsistency(self, docCodes, codes, checkCol2=True, checkCol3=True): + failed = 0 + ignored = 0 + # len + hLen = len(codes) + docLen = len(docCodes) + #errCodes = [] + #tdLog.info(f"head file codes items= {hLen} doc file codes items={docLen} \n") + for i in range(hLen): + problem = True + action = "not found" + for j in range(docLen): + if codes[i] == docCodes[j][0]: + #tdLog.info(f" i={i} error code: {codes[i]} found in doc code:{docCodes[j][0]}\n") + problem = False + if docCodes[j][1] == "": + # describe col1 must not empty + problem = True + action = "found but \"Error Description\" column is empty" + # check col2 empty + elif checkCol2 and docCodes[j][2] == "": + problem = True + action = "found but \"Possible Cause\" column is empty" + # check col3 empty + elif checkCol3 and docCodes[j][3] == "": + problem = True + action = "found but \"Suggested Actions\" column is empty" + + # found stop search next + break + + if problem: + # ignore + if self.ignoreCode(codes[i]): + #tdLog.info(f" i={i} error code: {codes[i]} ignore ...") + ignored += 1 + else: + tdLog.info(f" i={i} error code: {codes[i]} {action} in doc.") + failed +=1 + #errCodes.append(codes[i]) + + #print(errCodes) + # result + if failed > 0: + tdLog.exit("Failed to check the consistency of error codes between header and doc. " + f"failed:{failed}, ignored:{ignored}, all:{hLen}\n") + + tdLog.info(f"Check consistency successfully. ok items={hLen - ignored}, ignored items={ignored}\n") + + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + + # read head error code + hFile = "../../include/util/taoserror.h" + codes = self.readHeadCodes(hFile) + + # read zh codes + zhDoc = "../../docs/zh/14-reference/09-error-code.md" + zhCodes = self.readDocCodes(zhDoc) + + # read en codes + enDoc = "../../docs/en/14-reference/09-error-code.md" + enCodes = self.readDocCodes(enDoc) + + # check zh + tdLog.info(f"check zh docs ...") + self.checkConsistency(zhCodes, codes) + + # check en + tdLog.info(f"check en docs ...") + self.checkConsistency(enCodes, codes) + + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 055bd7f9c6..1791c3b325 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -53,6 +53,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f query/test_having.py ,,n,army,python3 ./test.py -f tmq/drop_lost_comsumers.py ,,y,army,./pytest.sh python3 ./test.py -f cmdline/taosCli.py +,,n,army,python3 ./test.py -f whole/checkErrorCode.py # # system test @@ -354,6 +355,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/telemetry.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/backquote_check.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdMonitor.py +,,n,system-test,python3 ./test.py -f 0-others/taosdlog.py ,,n,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 ,,n,system-test,python3 ./test.py -f 0-others/udfTest.py ,,n,system-test,python3 ./test.py -f 0-others/udf_create.py diff --git a/tests/system-test/0-others/taosdlog.py b/tests/system-test/0-others/taosdlog.py index d4698960cd..06ac4ce5e4 100644 --- a/tests/system-test/0-others/taosdlog.py +++ b/tests/system-test/0-others/taosdlog.py @@ -1,66 +1,351 @@ -import taos -import sys -import time -import os - -from util.log import * -from util.sql import * -from util.cases import * -from util.dnodes import * - -class TDTestCase: - - def init(self, conn, logSql, replicaVar=1): - self.replicaVar = int(replicaVar) - tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor()) - - def getBuildPath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files or "taosd.exe" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root) - len("/build/bin")] - break - return buildPath - - def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring - tdSql.prepare() - # time.sleep(2) - tdSql.query("create user testpy pass 'testpy'") - - buildPath = self.getBuildPath() - if (buildPath == ""): - tdLog.exit("taosd not found!") - else: - tdLog.info("taosd found in %s" % buildPath) - logPath = buildPath + "/../sim/dnode1/log" - tdLog.info("log path: %s" % logPath) - - tdDnodes.stop(1) - time.sleep(2) - tdSql.error("select * from information_schema.ins_databases") - os.system("rm -rf %s" % logPath) - if os.path.exists(logPath) == True: - tdLog.exit("log pat still exist!") - - tdDnodes.start(1) - time.sleep(2) - if os.path.exists(logPath) != True: - tdLog.exit("log pat is not generated!") - - tdSql.query("select * from information_schema.ins_databases") - - def stop(self): - tdSql.close() - tdLog.success(f"{__file__} successfully executed") - -tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) +import concurrent.futures +import os +import os.path +import platform +import shutil +import subprocess +import time + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + tdSql.prepare() + self.buildPath = self.getBuildPath() + if (self.buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % self.buildPath) + self.taosdPath = self.buildPath + "/build/bin/taosd" + self.taosPath = self.buildPath + "/build/bin/taos" + self.logPath = self.buildPath + "/../sim/dnode1/log" + tdLog.info("taosd path: %s" % self.taosdPath) + tdLog.info("taos path: %s" % self.taosPath) + tdLog.info("log path: %s" % self.logPath) + self.commonCfgDict = {} + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def prepareCfg(self, cfgPath, cfgDict): + tdLog.info("make dir %s" % cfgPath) + os.makedirs(cfgPath, exist_ok=True) + with open(cfgPath + "/taos.cfg", "w") as f: + for key in self.commonCfgDict: + f.write("%s %s\n" % (key, self.commonCfgDict[key])) + for key in cfgDict: + f.write("%s %s\n" % (key, cfgDict[key])) + if not os.path.exists(cfgPath + "/taos.cfg"): + tdLog.exit("taos.cfg not found in %s" % cfgPath) + else: + tdLog.info("taos.cfg found in %s" % cfgPath) + + def closeBin(self, binName): + tdLog.info("Closing %s" % binName) + if platform.system().lower() == 'windows': + psCmd = "for /f %%a in ('wmic process where \"name='%s.exe'\" get processId ^| xargs echo ^| awk ^'{print $2}^' ^&^& echo aa') do @(ps | grep %%a | awk '{print $1}' | xargs)" % binName + else: + psCmd = "ps -ef | grep -w %s | grep -v grep | awk '{print $2}'" % binName + tdLog.info(f"psCmd:{psCmd}") + + try: + rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + processID = rem.stdout.decode().strip() + tdLog.info(f"processID:{processID}") + except Exception as e: + tdLog.info(f"closeBin error:{e}") + processID = "" + onlyKillOnceWindows = 0 + while(processID): + if not platform.system().lower() == 'windows' or (onlyKillOnceWindows == 0 and platform.system().lower() == 'windows'): + killCmd = "kill -9 %s > /dev/null 2>&1" % processID + if platform.system().lower() == 'windows': + killCmd = "kill -9 %s > nul 2>&1" % processID + tdLog.info(f"killCmd:{killCmd}") + os.system(killCmd) + tdLog.info(f"killed {binName} process {processID}") + onlyKillOnceWindows = 1 + time.sleep(1) + try: + rem = subprocess.run(psCmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + processID = rem.stdout.decode().strip() + except Exception as e: + tdLog.info(f"closeBin error:{e}") + processID = "" + + def closeTaosd(self, signal=9): + self.closeBin("taosd") + + def closeTaos(self, signal=9): + self.closeBin("taos") + + def openBin(self, binPath, waitSec=5): + tdLog.info(f"Opening {binPath}") + try: + process = subprocess.Popen(binPath, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + time.sleep(waitSec) + if process.poll() is None: + tdLog.info(f"{binPath} started successfully") + else: + error = process.stderr.read().decode(encoding="utf-8").strip() + raise Exception(f"Failed to start {binPath}: {error}") + except Exception as e: + raise Exception(f"Failed to start {binPath}: %s" % repr(e)) + + def openTaosd(self, args="", waitSec=8): + self.openBin(f'{self.taosdPath} {args}', waitSec) + + def openTaos(self, args="", waitSec=5): + self.openBin(f'{self.taosPath} {args}', waitSec) + + + def prepare_logoutput(self, desc, port, logOutput, skipOpenBin=False): + tdLog.info("Preparing %s, port:%s" % (desc, port)) + dnodePath = self.buildPath + "/../sim/dnode%s" % port + tdLog.info("remove dnodePath:%s" % dnodePath) + try: + if os.path.exists(dnodePath): + shutil.rmtree(dnodePath) + except Exception as e: + raise Exception(f"Failed to remove directory {dnodePath}: {e}") + try: + self.prepareCfg(dnodePath, {"serverPort": port, + "dataDir": dnodePath + os.sep + "data", + "logDir": dnodePath + os.sep + "log"}) + except Exception as e: + raise Exception(f"Failed to prepare configuration for {dnodePath}: {e}") + try: + self.openTaosd(f"-c {dnodePath} {logOutput}") + self.openTaos(f"-c {dnodePath} {logOutput}") + except Exception as e: + if(skipOpenBin): + tdLog.info(f"Failed to prepare taosd and taos with log output {logOutput}: {e}") + else: + raise Exception(f"Failed to prepare taosd and taos with log output {logOutput}: {e}") + + + def prepare_stdout(self): + list = self.prepare_list[0] + self.prepare_logoutput(list[0], list[1], "-o " + list[0]) + + def check_stdout(self): + tdLog.info("Running check stdout") + dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[0][1] + tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0")) + tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0")) + + def prepare_stderr(self): + list = self.prepare_list[1] + self.prepare_logoutput(list[0], list[1], "--log-output " + list[0]) + + def check_stderr(self): + tdLog.info("Running check stderr") + dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[1][1] + tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0")) + tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0")) + + def prepare_dev_null(self): + list = self.prepare_list[2] + # self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0]) + + def check_dev_null(self): + tdLog.info("Running check /dev/null") + # dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[2][1] + # tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taosdlog.0")) + # tdSql.checkEqual(False, os.path.isfile(f"{dnodePath}/log/taoslog0.0")) + + def prepare_fullpath(self): + list = self.prepare_list[3] + dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[3][1] + self.prepare_logoutput(list[0], list[1], "-o " + dnodePath + "/log0/" ) + + def check_fullpath(self): + tdLog.info("Running check fullpath") + logPath = self.buildPath + "/../sim/dnode%s/log0/" % self.prepare_list[3][1] + tdSql.checkEqual(True, os.path.exists(f"{logPath}taosdlog.0")) + tdSql.checkEqual(True, os.path.exists(f"{logPath}taoslog0.0")) + + def prepare_fullname(self): + list = self.prepare_list[4] + dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[4][1] + self.prepare_logoutput(list[0], list[1], "--log-output " + dnodePath + "/log0/" + list[0]) + dnodePath = self.buildPath + "/../sim/dnode%s" % self.prepare_list[4][1] + + def check_fullname(self): + tdLog.info("Running check fullname") + logPath = self.buildPath + "/../sim/dnode%s/log0/" % self.prepare_list[4][1] + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[4][0] + ".0")) + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[4][0] + "0.0")) + + def prepare_relativepath(self): + list = self.prepare_list[5] + self.prepare_logoutput(list[0], list[1], "--log-output=" + "log0/") + + def check_relativepath(self): + tdLog.info("Running check relativepath") + logPath = self.buildPath + "/../sim/dnode%s/log/log0/" % self.prepare_list[5][1] + tdSql.checkEqual(True, os.path.exists(logPath + "taosdlog.0")) + tdSql.checkEqual(True, os.path.exists(logPath + "taoslog0.0")) + + def prepare_relativename(self): + list = self.prepare_list[6] + self.prepare_logoutput(list[0], list[1], "-o " + "log0/" + list[0]) + def check_relativename(self): + tdLog.info("Running check relativename") + logPath = self.buildPath + "/../sim/dnode%s/log/log0/" % self.prepare_list[6][1] + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[6][0] + ".0")) + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[6][0] + "0.0")) + + def prepare_filename(self): + list = self.prepare_list[7] + self.prepare_logoutput(list[0], list[1], "--log-output " + list[0]) + def check_filename(self): + tdLog.info("Running check filename") + logPath = self.buildPath + "/../sim/dnode%s/log/" % self.prepare_list[7][1] + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[7][0] + ".0")) + tdSql.checkEqual(True, os.path.exists(logPath + self.prepare_list[7][0] + "0.0")) + + def prepare_empty(self): + list = self.prepare_list[8] + self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0], True) + def check_empty(self): + tdLog.info("Running check empty") + logPath = self.buildPath + "/../sim/dnode%s/log" % self.prepare_list[8][1] + tdSql.checkEqual(False, os.path.exists(f"{logPath}/taosdlog.0")) + tdSql.checkEqual(False, os.path.exists(f"{logPath}/taoslog.0")) + + def prepare_illegal(self): + list = self.prepare_list[9] + self.prepare_logoutput(list[0], list[1], "--log-output=" + list[0], True) + def check_illegal(self): + tdLog.info("Running check empty") + logPath = self.buildPath + "/../sim/dnode%s/log" % self.prepare_list[9][1] + tdSql.checkEqual(False, os.path.exists(f"{logPath}/taosdlog.0")) + tdSql.checkEqual(False, os.path.exists(f"{logPath}/taoslog.0")) + + def prepareCheckResources(self): + self.prepare_list = [["stdout", "10030"], ["stderr", "10031"], ["/dev/null", "10032"], ["fullpath", "10033"], + ["fullname", "10034"], ["relativepath", "10035"], ["relativename", "10036"], ["filename", "10037"], + ["empty", "10038"], ["illeg?al", "10039"]] + self.check_functions = { + self.prepare_stdout: self.check_stdout, + self.prepare_stderr: self.check_stderr, + self.prepare_dev_null: self.check_dev_null, + self.prepare_fullpath: self.check_fullpath, + self.prepare_fullname: self.check_fullname, + self.prepare_relativepath: self.check_relativepath, + self.prepare_relativename: self.check_relativename, + self.prepare_filename: self.check_filename, + self.prepare_empty: self.check_empty, + self.prepare_illegal: self.check_illegal, + } + + def checkLogOutput(self): + self.closeTaosd() + self.closeTaos() + self.prepareCheckResources() + with concurrent.futures.ThreadPoolExecutor() as executor: + prepare_futures = [executor.submit(prepare_func) for prepare_func, _ in self.check_functions.items()] + for future in concurrent.futures.as_completed(prepare_futures): + try: + future.result() + except Exception as e: + raise Exception(f"Error in prepare function: {e}") + + check_futures = [executor.submit(check_func) for _, check_func in self.check_functions.items()] + for future in concurrent.futures.as_completed(check_futures): + try: + future.result() + except Exception as e: + raise Exception(f"Error in prepare function: {e}") + + def checkLogRotate(self): + tdLog.info("Running check log rotate") + dnodePath = self.buildPath + "/../sim/dnode10050" + logRotateAfterBoot = 6 # LOG_ROTATE_BOOT + self.closeTaosd() + self.closeTaos() + try: + if os.path.exists(dnodePath): + shutil.rmtree(dnodePath) + self.prepareCfg(dnodePath, {"serverPort": 10050, + "dataDir": dnodePath + os.sep + "data", + "logDir": dnodePath + os.sep + "log", + "logKeepDays": "3" }) + except Exception as e: + raise Exception(f"Failed to prepare configuration for {dnodePath}: {e}") + + nowSec = int(time.time()) + stubFile99 = f"{dnodePath}/log/taosdlog.99" + stubFile101 = f"{dnodePath}/log/taosdlog.101" + stubGzFile98 = f"{dnodePath}/log/taosdlog.98.gz" + stubGzFile102 = f"{dnodePath}/log/taosdlog.102.gz" + stubFileNow = f"{dnodePath}/log/taosdlog.{nowSec}" + stubGzFileNow = f"{dnodePath}/log/taosdlog.%d.gz" % (nowSec - 1) + stubGzFileKeep = f"{dnodePath}/log/taosdlog.%d.gz" % (nowSec - 86400 * 2) + stubGzFileDel = f"{dnodePath}/log/taosdlog.%d.gz" % (nowSec - 86400 * 3) + stubFiles = [stubFile99, stubFile101, stubGzFile98, stubGzFile102, stubFileNow, stubGzFileNow, stubGzFileKeep, stubGzFileDel] + + try: + os.makedirs(f"{dnodePath}/log", exist_ok=True) + for stubFile in stubFiles: + with open(stubFile, "w") as f: + f.write("test log rotate") + except Exception as e: + raise Exception(f"Failed to prepare log files for {dnodePath}: {e}") + + tdSql.checkEqual(True, os.path.exists(stubFile101)) + tdSql.checkEqual(True, os.path.exists(stubGzFile102)) + tdSql.checkEqual(True, os.path.exists(stubFileNow)) + tdSql.checkEqual(True, os.path.exists(stubGzFileDel)) + + self.openTaosd(f"-c {dnodePath}") + self.openTaos(f"-c {dnodePath}") + + tdLog.info("wait %d seconds for log rotate" % (logRotateAfterBoot + 2)) + time.sleep(logRotateAfterBoot + 2) + + tdSql.checkEqual(True, os.path.exists(stubFile99)) + tdSql.checkEqual(False, os.path.exists(stubFile101)) + tdSql.checkEqual(False, os.path.exists(f'{stubFile101}.gz')) + tdSql.checkEqual(True, os.path.exists(stubGzFile98)) + tdSql.checkEqual(True, os.path.exists(f'{stubFileNow}.gz')) + tdSql.checkEqual(True, os.path.exists(stubGzFileNow)) + tdSql.checkEqual(True, os.path.exists(stubGzFileKeep)) + tdSql.checkEqual(False, os.path.exists(stubGzFile102)) + tdSql.checkEqual(False, os.path.exists(stubGzFileDel)) + tdSql.checkEqual(True, os.path.exists(f"{dnodePath}/log/taosdlog.0")) + tdSql.checkEqual(True, os.path.exists(f"{dnodePath}/log/taoslog0.0")) + + def run(self): + self.checkLogOutput() + self.checkLogRotate() + self.closeTaosd() + self.closeTaos() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 442329674d..88918a14ac 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -49,7 +49,12 @@ #define SHELL_PKT_LEN "Packet length used for net test, default is 1024 bytes." #define SHELL_PKT_NUM "Packet numbers used for net test, default is 100." #define SHELL_BI_MODE "Set BI mode" -#define SHELL_VERSION "Print program version." +#define SHELL_LOG_OUTPUT \ + "Specify log output. Options:\n\r\t\t\t stdout, stderr, /dev/null, , /, " \ + "\n\r\t\t\t * If OUTPUT contains an absolute directory, logs will be stored in that directory " \ + "instead of logDir.\n\r\t\t\t * If OUTPUT contains a relative directory, logs will be stored in the directory " \ + "combined with logDir and the relative directory." +#define SHELL_VERSION "Print program version." #ifdef WEBSOCKET #define SHELL_DSN "Use dsn to connect to the cloud server or to a remote server which provides WebSocket connection." @@ -74,6 +79,9 @@ void shellPrintHelp() { printf("%s%s%s%s\r\n", indent, "-l,", indent, SHELL_PKT_LEN); printf("%s%s%s%s\r\n", indent, "-n,", indent, SHELL_NET_ROLE); printf("%s%s%s%s\r\n", indent, "-N,", indent, SHELL_PKT_NUM); +#if defined(LINUX) + printf("%s%s%s%s\r\n", indent, "-o,", indent, SHELL_LOG_OUTPUT); +#endif printf("%s%s%s%s\r\n", indent, "-p,", indent, SHELL_PASSWORD); printf("%s%s%s%s\r\n", indent, "-P,", indent, SHELL_PORT); printf("%s%s%s%s\r\n", indent, "-r,", indent, SHELL_RAW_TIME); @@ -134,6 +142,9 @@ static struct argp_option shellOptions[] = { #endif {"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM}, {"bimode", 'B', 0, 0, SHELL_BI_MODE}, +#if defined(LINUX) + {"log-output", 'o', "OUTPUT", 0, SHELL_LOG_OUTPUT}, +#endif {0}, }; @@ -222,6 +233,23 @@ static int32_t shellParseSingleOpt(int32_t key, char *arg) { case 'N': pArgs->pktNum = atoi(arg); break; +#if defined(LINUX) + case 'o': + if (strlen(arg) >= PATH_MAX) { + printf("failed to set log output since length overflow, max length is %d\n", PATH_MAX); + return TSDB_CODE_INVALID_CFG; + } + tsLogOutput = taosMemoryMalloc(PATH_MAX); + if (!tsLogOutput) { + printf("failed to set log output: '%s' since %s\n", arg, tstrerror(terrno)); + return terrno; + } + if (taosExpandDir(arg, tsLogOutput, PATH_MAX) != 0) { + printf("failed to expand log output: '%s' since %s\n", arg, tstrerror(terrno)); + return terrno; + } + break; +#endif #ifdef WEBSOCKET case 'R': pArgs->restful = true;