From 5fe99c5ad24a8d96b75716b18d12f6aedc1b4595 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sun, 23 Apr 2023 15:54:15 +0800 Subject: [PATCH] feat: support log long query --- include/common/tglobal.h | 8 ++++++ source/client/src/clientImpl.c | 2 +- source/common/src/tglobal.c | 44 +++++++++++++++++++++++++++++++ source/util/src/tlog.c | 48 +++++++++++++++++++++++++++++++--- 4 files changed, 98 insertions(+), 4 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 6fde7b48a2..25a386b23d 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -24,6 +24,12 @@ extern "C" { #endif +#define SLOW_LOG_TYPE_QUERY 0x1 +#define SLOW_LOG_TYPE_INSERT 0x2 +#define SLOW_LOG_TYPE_OTHERS 0x4 +#define SLOW_LOG_TYPE_ALL 0xFFFFFFFF + + // cluster extern char tsFirst[]; extern char tsSecond[]; @@ -118,6 +124,8 @@ extern int32_t tsRedirectFactor; extern int32_t tsRedirectMaxPeriod; extern int32_t tsMaxRetryWaitTime; extern bool tsUseAdapter; +extern int32_t tsSlowLogThreshold; +extern int32_t tsSlowLogScope; // client extern int32_t tsMinSlidingTime; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index ce174744ef..b9fa12b587 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1257,7 +1257,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t if (pRequest->code != TSDB_CODE_SUCCESS) { const char* errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(pRequest->code); - fprintf(stderr, "failed to connect to server, reason: %s\n\n", errorMsg); + tscError("failed to connect to server, reason: %s", errorMsg); terrno = pRequest->code; destroyRequest(pRequest); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index da4a912238..224b0068fe 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -117,6 +117,10 @@ int32_t tsRedirectFactor = 2; int32_t tsRedirectMaxPeriod = 1000; int32_t tsMaxRetryWaitTime = 10000; bool tsUseAdapter = false; +int32_t tsSlowLogThreshold = 3; // seconds +int32_t tsSlowLogScope = SLOW_LOG_TYPE_ALL; + + /* @@ -345,6 +349,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, true) != 0) return -1; if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, true) != 0) return -1; if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 0, INT32_MAX, true) != 0) return -1; + if (cfgAddString(pCfg, "slowLogScope", "", true) != 0) return -1; tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS); @@ -692,6 +698,36 @@ static void taosSetServerLogCfg(SConfig *pCfg) { metaDebugFlag = cfgGetItem(pCfg, "metaDebugFlag")->i32; } +static int32_t taosSetSlowLogScope(char *pScope) { + if (NULL == pScope || 0 == strlen(pScope)) { + tsSlowLogScope = SLOW_LOG_TYPE_ALL; + return 0; + } + + if (0 == strcasecmp(pScope, "all")) { + tsSlowLogScope = SLOW_LOG_TYPE_ALL; + return 0; + } + + if (0 == strcasecmp(pScope, "query")) { + tsSlowLogScope = SLOW_LOG_TYPE_QUERY; + return 0; + } + + if (0 == strcasecmp(pScope, "insert")) { + tsSlowLogScope = SLOW_LOG_TYPE_INSERT; + return 0; + } + + if (0 == strcasecmp(pScope, "others")) { + tsSlowLogScope = SLOW_LOG_TYPE_OTHERS; + return 0; + } + + uError("Invalid slowLog scope value:%s", pScope); + return -1; +} + static int32_t taosSetClientCfg(SConfig *pCfg) { tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; @@ -742,6 +778,10 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsUseAdapter = cfgGetItem(pCfg, "useAdapter")->bval; tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval; tsQueryMaxConcurrentTables = cfgGetItem(pCfg, "queryMaxConcurrentTables")->i64; + tsSlowLogThreshold = cfgGetItem(pCfg, "slowLogThreshold")->i32; + if (taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str)) { + return -1; + } tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32; @@ -1156,6 +1196,10 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { sDebugFlag = cfgGetItem(pCfg, "sDebugFlag")->i32; } else if (strcasecmp("smaDebugFlag", name) == 0) { smaDebugFlag = cfgGetItem(pCfg, "smaDebugFlag")->i32; + } else if (strcasecmp("slowLogThreshold", name) == 0) { + tsSlowLogThreshold = cfgGetItem(pCfg, "slowLogThreshold")->i32; + } else if (strcasecmp("slowLogScope", name) == 0) { + taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str) } break; } diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index a3d3c399ab..10ab389873 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -28,6 +28,7 @@ #define LOG_FILE_NAME_LEN 300 #define LOG_DEFAULT_BUF_SIZE (20 * 1024 * 1024) // 20MB +#define LOG_SLOW_BUF_SIZE (10 * 1024 * 1024) // 10MB #define LOG_DEFAULT_INTERVAL 25 #define LOG_INTERVAL_STEP 5 @@ -62,6 +63,7 @@ typedef struct { pid_t pid; char logName[LOG_FILE_NAME_LEN]; SLogBuff *logHandle; + SLogBuff *slowHandle; TdThreadMutex logMutex; } SLogObj; @@ -136,6 +138,34 @@ static int32_t taosStartLog() { return 0; } +int32_t taosInitSlowLog() { + char fullName[PATH_MAX] = {0}; + char logFileName[64] = {0}; +#ifdef CUS_PROMPT + snprintf(logFileName, 64, "%sSlowLog", CUS_PROMPT); +#else + snprintf(logFileName, 64, "taosSlowLog"); +#endif + + if (strlen(tsLogDir) != 0) { + snprintf(fullName, PATH_MAX, "%s" TD_DIRSEP "%s", tsLogDir, logFileName); + } else { + snprintf(fullName, PATH_MAX, "%s", logFileName); + } + + tsLogObj.slowHandle = taosLogBuffNew(LOG_SLOW_BUF_SIZE); + if (tsLogObj.slowHandle == NULL) return -1; + + taosUmaskFile(0); + tsLogObj.slowHandle->pFile = taosOpenFile(fullName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (tsLogObj.slowHandle->pFile == NULL) { + printf("\nfailed to open slow log file:%s, reason:%s\n", fullName, strerror(errno)); + return -1; + } + + return 0; +} + int32_t taosInitLog(const char *logName, int32_t maxFiles) { if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0; osUpdate(); @@ -151,6 +181,8 @@ int32_t taosInitLog(const char *logName, int32_t maxFiles) { tsLogObj.logHandle = taosLogBuffNew(LOG_DEFAULT_BUF_SIZE); if (tsLogObj.logHandle == NULL) return -1; if (taosOpenLogFile(fullName, tsNumOfLogLines, maxFiles) < 0) return -1; + + if (taosInitSlowLog() < 0) return -1; if (taosStartLog() < 0) return -1; return 0; } @@ -159,11 +191,23 @@ static void taosStopLog() { if (tsLogObj.logHandle) { tsLogObj.logHandle->stop = 1; } + if (tsLogObj.slowHandle) { + tsLogObj.slowHandle->stop = 1; + } } void taosCloseLog() { + taosStopLog(); + + if (tsLogObj.slowHandle != NULL) { + taosThreadMutexDestroy(&tsLogObj.slowHandle->buffMutex); + taosCloseFile(&tsLogObj.slowHandle->pFile); + taosMemoryFreeClear(tsLogObj.slowHandle->buffer); + memset(&tsLogObj.slowHandle->buffer, 0, sizeof(tsLogObj.slowHandle->buffer)); + taosMemoryFreeClear(tsLogObj.slowHandle); + } + if (tsLogObj.logHandle != NULL) { - taosStopLog(); if (tsLogObj.logHandle != NULL && taosCheckPthreadValid(tsLogObj.logHandle->asyncThread)) { taosThreadJoin(tsLogObj.logHandle->asyncThread, NULL); taosThreadClear(&tsLogObj.logHandle->asyncThread); @@ -176,8 +220,6 @@ void taosCloseLog() { memset(&tsLogObj.logHandle->buffer, 0, sizeof(tsLogObj.logHandle->buffer)); taosThreadMutexDestroy(&tsLogObj.logMutex); taosMemoryFreeClear(tsLogObj.logHandle); - memset(&tsLogObj.logHandle, 0, sizeof(tsLogObj.logHandle)); - tsLogObj.logHandle = NULL; } }