From 09cf91f256a9c64f87ac928a093ab83b7ac927fb Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 10 Oct 2024 14:52:42 +0800 Subject: [PATCH] tcs/dnode: use tcs interface for dnode --- source/common/src/tglobal.c | 59 ++++++++++++---------- source/dnode/mgmt/exe/dmMain.c | 6 +-- source/dnode/mgmt/node_mgmt/CMakeLists.txt | 2 +- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 18 +++---- 4 files changed, 42 insertions(+), 43 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index af1a8ccfbe..45546f9a59 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -179,12 +179,12 @@ int32_t tsRedirectFactor = 2; int32_t tsRedirectMaxPeriod = 1000; int32_t tsMaxRetryWaitTime = 10000; bool tsUseAdapter = false; -int32_t tsMetaCacheMaxSize = -1; // MB -int32_t tsSlowLogThreshold = 10; // seconds -int32_t tsSlowLogThresholdTest = INT32_MAX; // seconds -char tsSlowLogExceptDb[TSDB_DB_NAME_LEN] = ""; // seconds +int32_t tsMetaCacheMaxSize = -1; // MB +int32_t tsSlowLogThreshold = 10; // seconds +int32_t tsSlowLogThresholdTest = INT32_MAX; // seconds +char tsSlowLogExceptDb[TSDB_DB_NAME_LEN] = ""; // seconds int32_t tsSlowLogScope = SLOW_LOG_TYPE_QUERY; -char* tsSlowLogScopeString = "query"; +char *tsSlowLogScopeString = "query"; int32_t tsSlowLogMaxLen = 4096; int32_t tsTimeSeriesThreshold = 50; bool tsMultiResultFunctionStarReturnTags = false; @@ -306,6 +306,7 @@ char tsS3AppId[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {""}; int8_t tsS3Enabled = false; int8_t tsS3EnabledCfg = false; int8_t tsS3Oss[TSDB_MAX_EP_NUM] = {false}; +int8_t tsS3Ablob = false; int8_t tsS3StreamEnabled = false; int8_t tsS3Https[TSDB_MAX_EP_NUM] = {true}; @@ -322,7 +323,6 @@ int32_t tsMaxTsmaNum = 3; int32_t tsMaxTsmaCalcDelay = 600; int64_t tsmaDataDeleteMark = 1000 * 60 * 60 * 24; // in ms, default to 1d - #define TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, pName) \ if ((pItem = cfgGetItem(pCfg, pName)) == NULL) { \ TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); \ @@ -361,7 +361,7 @@ static int32_t taosSplitS3Cfg(SConfig *pCfg, const char *name, char gVarible[TSD TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, name); char *strDup = NULL; - if ((strDup = taosStrdup(pItem->str))== NULL){ + if ((strDup = taosStrdup(pItem->str)) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -435,6 +435,7 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { } tsS3Https[i] = (strstr(tsS3Endpoint[i], "https://") != NULL); tsS3Oss[i] = (strstr(tsS3Endpoint[i], "aliyuncs.") != NULL); + tsS3Ablob = (strstr(tsS3Endpoint[i], ".blob.core.windows.net") != NULL); } if (tsS3BucketName[0] != '<') { @@ -450,7 +451,9 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { TAOS_RETURN(TSDB_CODE_SUCCESS); } -struct SConfig *taosGetCfg() { return tsCfg; } +struct SConfig *taosGetCfg() { + return tsCfg; +} static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, char *apolloUrl) { @@ -572,7 +575,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { TAOS_CHECK_RETURN( cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, CFG_SCOPE_CLIENT, CFG_DYN_ENT_CLIENT)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "queryTableNotExistAsEmpty", tsQueryTbNotExistAsEmpty, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); + TAOS_CHECK_RETURN( + cfgAddBool(pCfg, "queryTableNotExistAsEmpty", tsQueryTbNotExistAsEmpty, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableQueryHb", tsEnableQueryHb, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableScience", tsEnableScience, CFG_SCOPE_CLIENT, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); @@ -600,7 +604,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { TAOS_CHECK_RETURN( cfgAddInt32(pCfg, "metaCacheMaxSize", tsMetaCacheMaxSize, -1, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "randErrorChance", tsRandErrChance, 0, 10000, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorDivisor", tsRandErrDivisor, 1, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); + TAOS_CHECK_RETURN( + cfgAddInt64(pCfg, "randErrorDivisor", tsRandErrDivisor, 1, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorScope", tsRandErrScope, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); tsNumOfRpcThreads = tsNumOfCores / 2; @@ -1088,9 +1093,9 @@ int32_t taosSetSlowLogScope(char *pScopeStr, int32_t *pScope) { int32_t slowScope = 0; - char* scope = NULL; - char *tmp = NULL; - while((scope = strsep(&pScopeStr, "|")) != NULL){ + char *scope = NULL; + char *tmp = NULL; + while ((scope = strsep(&pScopeStr, "|")) != NULL) { taosMemoryFreeClear(tmp); tmp = taosStrdup(scope); if (tmp == NULL) { @@ -1147,13 +1152,13 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { (void)snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "firstEp"); - SEp firstEp = {0}; + SEp firstEp = {0}; TAOS_CHECK_RETURN(taosGetFqdnPortFromEp(strlen(pItem->str) == 0 ? defaultFirstEp : pItem->str, &firstEp)); (void)snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port); TAOS_CHECK_RETURN(cfgSetItem(pCfg, "firstEp", tsFirst, pItem->stype, true)); TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "secondEp"); - SEp secondEp = {0}; + SEp secondEp = {0}; TAOS_CHECK_RETURN(taosGetFqdnPortFromEp(strlen(pItem->str) == 0 ? defaultFirstEp : pItem->str, &secondEp)); (void)snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port); TAOS_CHECK_RETURN(cfgSetItem(pCfg, "secondEp", tsSecond, pItem->stype, true)); @@ -1653,8 +1658,8 @@ static int32_t taosSetAllDebugFlag(SConfig *pCfg, int32_t flag); int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SConfig *pCfg = NULL; if (tsCfg == NULL) { @@ -1725,7 +1730,7 @@ int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char * TAOS_CHECK_RETURN(cfgInit(&pCfg)); TAOS_CHECK_GOTO(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE), NULL, _exit); - TAOS_CHECK_GOTO(cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) ,NULL, _exit); + TAOS_CHECK_GOTO(cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER), NULL, _exit); if ((code = taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) { (void)printf("failed to load cfg since %s\n", tstrerror(code)); @@ -1754,7 +1759,7 @@ _exit: static int32_t taosCheckGlobalCfg() { uint32_t ipv4 = 0; - int32_t code = taosGetIpv4FromFqdn(tsLocalFqdn, &ipv4); + int32_t code = taosGetIpv4FromFqdn(tsLocalFqdn, &ipv4); if (code) { uError("failed to get ip from fqdn:%s since %s, dnode can not be initialized", tsLocalFqdn, tstrerror(code)); TAOS_RETURN(TSDB_CODE_RPC_FQDN_ERROR); @@ -1859,7 +1864,7 @@ typedef struct { static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize, SConfigItem *pItem, bool isDebugflag) { int32_t code = TSDB_CODE_CFG_NOT_FOUND; - char *name = pItem->name; + char *name = pItem->name; for (int32_t d = 0; d < optionSize; ++d) { const char *optName = pOptions[d].optionName; if (strcasecmp(name, optName) != 0) continue; @@ -2054,8 +2059,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { } case 'f': { if (strcasecmp("fqdn", name) == 0) { - SConfigItem* pFqdnItem = cfgGetItem(pCfg, "fqdn"); - SConfigItem* pServerPortItem = cfgGetItem(pCfg, "serverPort"); + SConfigItem *pFqdnItem = cfgGetItem(pCfg, "fqdn"); + SConfigItem *pServerPortItem = cfgGetItem(pCfg, "serverPort"); SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp"); if (pFqdnItem == NULL || pServerPortItem == NULL || pFirstEpItem == NULL) { uError("failed to get fqdn or serverPort or firstEp from cfg"); @@ -2070,7 +2075,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { char defaultFirstEp[TSDB_EP_LEN] = {0}; (void)snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); - SEp firstEp = {0}; + SEp firstEp = {0}; TAOS_CHECK_GOTO( taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp), &lino, _out); @@ -2110,8 +2115,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { } case 'l': { if (strcasecmp("locale", name) == 0) { - SConfigItem* pLocaleItem = cfgGetItem(pCfg, "locale"); - SConfigItem* pCharsetItem = cfgGetItem(pCfg, "charset"); + SConfigItem *pLocaleItem = cfgGetItem(pCfg, "locale"); + SConfigItem *pCharsetItem = cfgGetItem(pCfg, "charset"); if (pLocaleItem == NULL || pCharsetItem == NULL) { uError("failed to get locale or charset from cfg"); code = TSDB_CODE_CFG_NOT_FOUND; @@ -2184,7 +2189,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { char defaultFirstEp[TSDB_EP_LEN] = {0}; (void)snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); - SEp firstEp = {0}; + SEp firstEp = {0}; TAOS_CHECK_GOTO( taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp), &lino, _out); @@ -2315,7 +2320,7 @@ int32_t taosSetGlobalDebugFlag(int32_t flag) { return taosSetAllDebugFlag(tsCfg, // NOTE: set all command does not change the tmrDebugFlag static int32_t taosSetAllDebugFlag(SConfig *pCfg, int32_t flag) { if (flag < 0) TAOS_RETURN(TSDB_CODE_INVALID_PARA); - if (flag == 0) TAOS_RETURN(TSDB_CODE_SUCCESS); // just ignore + if (flag == 0) TAOS_RETURN(TSDB_CODE_SUCCESS); // just ignore SArray *noNeedToSetVars = NULL; SConfigItem *pItem = NULL; diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 89569d69d6..6069dc33f1 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -23,6 +23,7 @@ #include "jemalloc/jemalloc.h" #endif #include "dmUtil.h" +#include "tcs.h" #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #include "cus_name.h" @@ -325,10 +326,9 @@ static int32_t dmCheckS3() { int32_t code = 0; SConfig *pCfg = taosGetCfg(); cfgDumpCfgS3(pCfg, 0, true); -#if defined(USE_S3) - extern int32_t s3CheckCfg(); - code = s3CheckCfg(); +#if defined(USE_S3) + code = tcsCheckCfg(); #endif return code; } diff --git a/source/dnode/mgmt/node_mgmt/CMakeLists.txt b/source/dnode/mgmt/node_mgmt/CMakeLists.txt index 82b9384d66..98de62eee1 100644 --- a/source/dnode/mgmt/node_mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/node_mgmt/CMakeLists.txt @@ -1,7 +1,7 @@ aux_source_directory(src IMPLEMENT_SRC) add_library(dnode STATIC ${IMPLEMENT_SRC}) target_link_libraries( - dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode monitorfw + dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode monitorfw tcs ) IF (TD_ENTERPRISE) diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 2d0ad70adf..d72bc79034 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -20,6 +20,7 @@ #include "libs/function/tudf.h" #include "tgrant.h" #include "tcompare.h" +#include "tcs.h" // clang-format on #define DM_INIT_AUDIT() \ @@ -97,9 +98,9 @@ static bool dmDataSpaceAvailable() { static int32_t dmCheckDiskSpace() { // availability int32_t code = 0; - code = osUpdate(); - if(code != 0) { - code = 0; // ignore the error, just log it + code = osUpdate(); + if (code != 0) { + code = 0; // ignore the error, just log it dError("failed to update os info since %s", tstrerror(code)); } if (!dmDataSpaceAvailable()) { @@ -162,13 +163,6 @@ static int32_t dmCheckDataDirVersionWrapper() { } return 0; } -#if defined(USE_S3) - -extern int32_t s3Begin(); -extern void s3End(); -extern int8_t tsS3Enabled; - -#endif int32_t dmInit() { dInfo("start to init dnode env"); @@ -186,7 +180,7 @@ int32_t dmInit() { if ((code = dmInitDnode(dmInstance())) != 0) return code; if ((code = InitRegexCache() != 0)) return code; #if defined(USE_S3) - if ((code = s3Begin()) != 0) return code; + if ((code = tcsInit()) != 0) return code; #endif dInfo("dnode env is initialized"); @@ -219,7 +213,7 @@ void dmCleanup() { DestroyRegexCache(); #if defined(USE_S3) - s3End(); + tcsUninit(); #endif dInfo("dnode env is cleaned up");