From cda8bd8047b8e00526f5164976ebb4b36701658c Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 21 Oct 2024 13:51:33 +0800 Subject: [PATCH 01/99] Add category in configuration's struct --- include/util/tconfig.h | 29 +- source/common/src/tglobal.c | 512 +++++++++++++++++++---------------- source/util/src/tconfig.c | 23 +- source/util/test/cfgTest.cpp | 12 +- 4 files changed, 323 insertions(+), 253 deletions(-) diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 3fc247982f..d687d59ac2 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -52,20 +52,29 @@ typedef enum { } ECfgDataType; typedef enum { CFG_SCOPE_SERVER, CFG_SCOPE_CLIENT, CFG_SCOPE_BOTH } ECfgScopeType; +typedef enum { CFG_CATEGORY_GLOBAL, CFG_CATEGORY_LOCAL } ECfgCategoryType; +typedef enum { + CFG_DYNAMIC_MODIFICATION_SUPPORT, + CFG_DYNAMIC_MODIFICATION_NOT_SUPPORT, + CFG_MODIFICATION_READONLY +} ECfgModificationType; typedef enum { CFG_DYN_NONE = 0, CFG_DYN_SERVER = 1, CFG_DYN_CLIENT = 2, CFG_DYN_BOTH = 3, + CFG_DYN_READONLY = 4, #ifdef TD_ENTERPRISE CFG_DYN_ENT_SERVER = CFG_DYN_SERVER, CFG_DYN_ENT_CLIENT = CFG_DYN_CLIENT, CFG_DYN_ENT_BOTH = CFG_DYN_BOTH, + CFG_DYN_ENT_READONLY = CFG_DYN_READONLY, #else CFG_DYN_ENT_SERVER = CFG_DYN_NONE, CFG_DYN_ENT_CLIENT = CFG_DYN_NONE, CFG_DYN_ENT_BOTH = CFG_DYN_NONE, + CFG_DYN_READONLY = CFG_DYN_NONE, #endif } ECfgDynType; @@ -98,7 +107,7 @@ typedef struct { const char *value; } SConfigPair; -typedef struct SConfig SConfig; +typedef struct SConfig SConfig; typedef struct SConfigIter SConfigIter; int32_t cfgInit(SConfig **ppCfg); @@ -117,15 +126,15 @@ void cfgLock(SConfig *pCfg); void cfgUnLock(SConfig *pCfg); // clang-format off -int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope, int8_t dynScope); -int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, int8_t dynScope); -int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, int8_t dynScope); -int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope, int8_t dynScope); -int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope); -int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope); -int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope); -int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope); -int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope); +int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope, int8_t dynScope,int8_t category); +int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, int8_t dynScope,int8_t category); +int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, int8_t dynScope,int8_t category); +int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope, int8_t dynScope,int8_t category); +int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope,int8_t category); +int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope,int8_t category); +int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope,int8_t category); +int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope,int8_t category); +int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope,int8_t category); // clang-format on const char *cfgStypeStr(ECfgSrcType type); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 3c05294264..9a63ab5013 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -14,10 +14,10 @@ */ #define _DEFAULT_SOURCE -#include "tglobal.h" #include "defines.h" #include "os.h" #include "tconfig.h" +#include "tglobal.h" #include "tgrant.h" #include "tlog.h" #include "tmisce.h" @@ -179,12 +179,12 @@ int32_t tsRedirectFactor = 2; int32_t tsRedirectMaxPeriod = 1000; int32_t tsMaxRetryWaitTime = 10000; bool tsUseAdapter = false; -int32_t tsMetaCacheMaxSize = -1; // MB -int32_t tsSlowLogThreshold = 10; // seconds -int32_t tsSlowLogThresholdTest = INT32_MAX; // seconds -char tsSlowLogExceptDb[TSDB_DB_NAME_LEN] = ""; // seconds +int32_t tsMetaCacheMaxSize = -1; // MB +int32_t tsSlowLogThreshold = 10; // seconds +int32_t tsSlowLogThresholdTest = INT32_MAX; // seconds +char tsSlowLogExceptDb[TSDB_DB_NAME_LEN] = ""; // seconds int32_t tsSlowLogScope = SLOW_LOG_TYPE_QUERY; -char* tsSlowLogScopeString = "query"; +char *tsSlowLogScopeString = "query"; int32_t tsSlowLogMaxLen = 4096; int32_t tsTimeSeriesThreshold = 50; bool tsMultiResultFunctionStarReturnTags = false; @@ -322,7 +322,6 @@ int32_t tsMaxTsmaNum = 3; int32_t tsMaxTsmaCalcDelay = 600; int64_t tsmaDataDeleteMark = 1000 * 60 * 60 * 24; // in ms, default to 1d - #define TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, pName) \ if ((pItem = cfgGetItem(pCfg, pName)) == NULL) { \ TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); \ @@ -361,7 +360,7 @@ static int32_t taosSplitS3Cfg(SConfig *pCfg, const char *name, char gVarible[TSD TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, name); char *strDup = NULL; - if ((strDup = taosStrdup(pItem->str))== NULL){ + if ((strDup = taosStrdup(pItem->str)) == NULL) { code = terrno; goto _exit; } @@ -515,42 +514,67 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input } int32_t taosAddClientLogCfg(SConfig *pCfg) { - TAOS_CHECK_RETURN(cfgAddDir(pCfg, "configDir", configDir, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddDir(pCfg, "scriptDir", configDir, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddDir(pCfg, "logDir", tsLogDir, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); + TAOS_CHECK_RETURN(cfgAddDir(pCfg, "configDir", configDir, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddDir(pCfg, "scriptDir", configDir, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddDir(pCfg, "logDir", tsLogDir, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN( - cfgAddInt32(pCfg, "numOfLogLines", tsNumOfLogLines, 1000, 2000000000, CFG_SCOPE_BOTH, CFG_DYN_ENT_BOTH)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "asyncLog", tsAsyncLog, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000, CFG_SCOPE_BOTH, CFG_DYN_ENT_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "simDebugFlag", simDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "jniDebugFlag", jniDebugFlag, 0, 255, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); + cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfLogLines", tsNumOfLogLines, 1000, 2000000000, CFG_SCOPE_BOTH, + CFG_DYN_ENT_BOTH, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "asyncLog", tsAsyncLog, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000, CFG_SCOPE_BOTH, CFG_DYN_ENT_BOTH, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "simDebugFlag", simDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "jniDebugFlag", jniDebugFlag, 0, 255, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_RETURN(TSDB_CODE_SUCCESS); } static int32_t taosAddServerLogCfg(SConfig *pCfg) { - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "vDebugFlag", vDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "mDebugFlag", mDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "wDebugFlag", wDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqClientDebug", tqClientDebug, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "udfDebugFlag", udfDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "idxDebugFlag", idxDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tdbDebugFlag", tdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "metaDebugFlag", metaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "stDebugFlag", stDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "sndDebugFlag", sndDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "vDebugFlag", vDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "mDebugFlag", mDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "wDebugFlag", wDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "tqClientDebug", tqClientDebug, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "udfDebugFlag", udfDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "idxDebugFlag", idxDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "tdbDebugFlag", tdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "metaDebugFlag", metaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "stDebugFlag", stDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "sndDebugFlag", sndDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -561,110 +585,139 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { (void)strcpy(defaultFqdn, "localhost"); } - TAOS_CHECK_RETURN(cfgAddString(pCfg, "firstEp", "", CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "secondEp", "", CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "fqdn", defaultFqdn, CFG_SCOPE_SERVER, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddDir(pCfg, "tempDir", tsTempDir, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "minimalTmpDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "firstEp", "", CFG_SCOPE_BOTH, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "secondEp", "", CFG_SCOPE_BOTH, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "fqdn", defaultFqdn, CFG_SCOPE_SERVER, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_CLIENT, + CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddDir(pCfg, "tempDir", tsTempDir, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN( - cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); + cfgAddFloat(pCfg, "minimalTmpDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, CFG_SCOPE_BOTH, CFG_DYN_BOTH, + CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, CFG_SCOPE_BOTH, CFG_DYN_BOTH, + CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN( - cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, CFG_SCOPE_CLIENT, CFG_DYN_ENT_CLIENT)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "queryTableNotExistAsEmpty", tsQueryTbNotExistAsEmpty, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableQueryHb", tsEnableQueryHb, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableScience", tsEnableScience, CFG_SCOPE_CLIENT, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); + cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, CFG_SCOPE_CLIENT, CFG_DYN_ENT_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "queryTableNotExistAsEmpty", tsQueryTbNotExistAsEmpty, CFG_SCOPE_CLIENT, + CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddBool(pCfg, "enableQueryHb", tsEnableQueryHb, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddBool(pCfg, "enableScience", tsEnableScience, CFG_SCOPE_CLIENT, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, + CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, CFG_SCOPE_CLIENT, - CFG_DYN_CLIENT)); + CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, + CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN( - cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "keepColumnName", tsKeepColumnName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "smlChildTableName", tsSmlChildTableName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); + cfgAddBool(pCfg, "keepColumnName", tsKeepColumnName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "smlChildTableName", tsSmlChildTableName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, + CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "smlAutoChildTableNameDelimiter", tsSmlAutoChildTableNameDelimiter, - CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "smlTagName", tsSmlTagName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "smlTsDefaultName", tsSmlTsDefaultName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "smlDot2Underline", tsSmlDot2Underline, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, CFG_SCOPE_CLIENT, CFG_DYN_NONE)); + CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddString(pCfg, "smlTagName", tsSmlTagName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddString(pCfg, "smlTsDefaultName", tsSmlTsDefaultName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddBool(pCfg, "smlDot2Underline", tsSmlDot2Underline, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, CFG_SCOPE_CLIENT, CFG_DYN_BOTH, + CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxInsertBatchRows", tsMaxInsertBatchRows, 1, INT32_MAX, CFG_SCOPE_CLIENT, - CFG_DYN_CLIENT) != 0); + CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL) != 0); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_BOTH, CFG_DYN_BOTH, + CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "useAdapter", tsUseAdapter, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN( - cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "useAdapter", tsUseAdapter, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); + cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, - CFG_SCOPE_CLIENT, CFG_DYN_NONE)); - TAOS_CHECK_RETURN( - cfgAddInt32(pCfg, "metaCacheMaxSize", tsMetaCacheMaxSize, -1, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "randErrorChance", tsRandErrChance, 0, 10000, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorDivisor", tsRandErrDivisor, 1, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorScope", tsRandErrScope, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); + CFG_SCOPE_CLIENT, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "metaCacheMaxSize", tsMetaCacheMaxSize, -1, INT32_MAX, CFG_SCOPE_CLIENT, + CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "randErrorChance", tsRandErrChance, 0, 10000, CFG_SCOPE_BOTH, CFG_DYN_BOTH, + CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorDivisor", tsRandErrDivisor, 1, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH, + CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorScope", tsRandErrScope, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH, + CFG_CATEGORY_GLOBAL)); tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH, CFG_DYN_BOTH, + CFG_CATEGORY_GLOBAL)); tsNumOfRpcSessions = TRANGE(tsNumOfRpcSessions, 100, 100000); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_BOTH, + CFG_CATEGORY_GLOBAL)); tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 10000000); - TAOS_CHECK_RETURN( - cfgAddInt32(pCfg, "timeToGetAvailableConn", tsTimeToGetAvailableConn, 20, 1000000, CFG_SCOPE_BOTH, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "timeToGetAvailableConn", tsTimeToGetAvailableConn, 20, 1000000, CFG_SCOPE_BOTH, + CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); tsKeepAliveIdle = TRANGE(tsKeepAliveIdle, 1, 72000); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "keepAliveIdle", tsKeepAliveIdle, 1, 7200000, CFG_SCOPE_BOTH, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "keepAliveIdle", tsKeepAliveIdle, 1, 7200000, CFG_SCOPE_BOTH, CFG_DYN_BOTH, + CFG_CATEGORY_GLOBAL)); tsNumOfTaskQueueThreads = tsNumOfCores * 2; tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 16); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT, + CFG_DYN_CLIENT, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN( - cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); + cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "multiResultFunctionStarReturnTags", tsMultiResultFunctionStarReturnTags, - CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN( - cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN( - cfgAddInt32(pCfg, "maxTsmaCalcDelay", tsMaxTsmaCalcDelay, 600, 86400, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); + CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, CFG_SCOPE_BOTH, + CFG_DYN_CLIENT, CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxTsmaCalcDelay", tsMaxTsmaCalcDelay, 600, 86400, CFG_SCOPE_CLIENT, + CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tsmaDataDeleteMark", tsmaDataDeleteMark, 60 * 60 * 1000, INT64_MAX, - CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); + CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_RETURN(TSDB_CODE_SUCCESS); } static int32_t taosAddSystemCfg(SConfig *pCfg) { SysNameInfo info = taosGetSysNameInfo(); - TAOS_CHECK_RETURN(cfgAddTimezone(pCfg, "timezone", tsTimezoneStr, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddLocale(pCfg, "locale", tsLocale, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddCharset(pCfg, "charset", tsCharset, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "assert", tsAssert, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableCoreFile", tsEnableCoreFile, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddTimezone(pCfg, "timezone", tsTimezoneStr, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddLocale(pCfg, "locale", tsLocale, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddCharset(pCfg, "charset", tsCharset, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN( + cfgAddBool(pCfg, "enableCoreFile", tsEnableCoreFile, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN( + cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "simdEnable", tsSIMDEnable, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "AVX512Enable", tsAVX512Enable, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "tagFilterCache", tsTagFilterCache, CFG_SCOPE_BOTH, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "simdEnable", tsSIMDEnable, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "AVX512Enable", tsAVX512Enable, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddBool(pCfg, "tagFilterCache", tsTagFilterCache, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE)); + TAOS_CHECK_RETURN( + cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); #if !defined(_ALPINE) - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE)); + TAOS_CHECK_RETURN( + cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); #endif - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "pageSizeKB", tsPageSizeKB, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "totalMemoryKB", tsTotalMemoryKB, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "os sysname", info.sysname, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "os nodename", info.nodename, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "os release", info.release, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "os version", info.version, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "os machine", info.machine, CFG_SCOPE_BOTH, CFG_DYN_NONE)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "pageSizeKB", tsPageSizeKB, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "totalMemoryKB", tsTotalMemoryKB, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE, + CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "os sysname", info.sysname, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "os nodename", info.nodename, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "os release", info.release, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "os version", info.version, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "os machine", info.machine, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "version", version, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "compatible_version", compatible_version, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "gitinfo", gitinfo, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "buildinfo", buildinfo, CFG_SCOPE_BOTH, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "version", version, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddString(pCfg, "compatible_version", compatible_version, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "gitinfo", gitinfo, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "buildinfo", buildinfo, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -703,137 +756,137 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsLogBufferMemoryAllowed = TRANGE(tsLogBufferMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL); // clang-format off - TAOS_CHECK_RETURN(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); + TAOS_CHECK_RETURN(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER, CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "encryptAlgorithm", tsEncryptAlgorithm, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "encryptScope", tsEncryptScope, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "encryptAlgorithm", tsEncryptAlgorithm, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "encryptScope", tsEncryptScope, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "retentionSpeedLimitMB", tsRetentionSpeedLimitMB, 0, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "retentionSpeedLimitMB", tsRetentionSpeedLimitMB, 0, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 4, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 4, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfVnodeRsmaThreads", tsNumOfVnodeRsmaThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfVnodeRsmaThreads", tsNumOfVnodeRsmaThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16, (TSDB_SYNC_SNAP_BUFFER_SIZE >> 2), CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "syncLogBufferMemoryAllowed", tsLogBufferMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16, (TSDB_SYNC_SNAP_BUFFER_SIZE >> 2), CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "syncLogBufferMemoryAllowed", tsLogBufferMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "arbHeartBeatIntervalSec", tsArbHeartBeatIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "arbCheckSyncIntervalSec", tsArbCheckSyncIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "arbSetAssignedTimeoutSec", tsArbSetAssignedTimeoutSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "arbHeartBeatIntervalSec", tsArbHeartBeatIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "arbCheckSyncIntervalSec", tsArbCheckSyncIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "arbSetAssignedTimeoutSec", tsArbSetAssignedTimeoutSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 86400, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 86400, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "slowLogThresholdTest", tsSlowLogThresholdTest, 0, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 1, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "slowLogMaxLen", tsSlowLogMaxLen, 1, 16384, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "slowLogScope", tsSlowLogScopeString, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "slowLogExceptDb", tsSlowLogExceptDb, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "slowLogThresholdTest", tsSlowLogThresholdTest, 0, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 1, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "slowLogMaxLen", tsSlowLogMaxLen, 1, 16384, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "slowLogScope", tsSlowLogScopeString, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "slowLogExceptDb", tsSlowLogExceptDb, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitorLogProtocol", tsMonitorLogProtocol, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitorForceV2", tsMonitorForceV2, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitorLogProtocol", tsMonitorLogProtocol, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitorForceV2", tsMonitorForceV2, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "auditInterval", tsAuditInterval, 500, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "auditInterval", tsAuditInterval, 500, 200000, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_BOTH, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "telemetryInterval", tsTelemInterval, 1, 200000, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, CFG_SCOPE_BOTH, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_BOTH, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "telemetryInterval", tsTelemInterval, 1, 200000, CFG_SCOPE_BOTH, CFG_DYN_BOTH,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_BOTH, CFG_DYN_BOTH,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, CFG_SCOPE_BOTH, CFG_DYN_NONE,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "rsyncPort", tsRsyncPort, 1, 65535, CFG_SCOPE_BOTH, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "snodeAddress", tsSnodeAddress, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "rsyncPort", tsRsyncPort, 1, 65535, CFG_SCOPE_BOTH, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "snodeAddress", tsSnodeAddress, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tmqRowSize", tmqRowSize, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tmqRowSize", tmqRowSize, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxTsmaNum", tsMaxTsmaNum, 0, 3, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxTsmaNum", tsMaxTsmaNum, 0, 3, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlBatchDropNum", tsTtlBatchDropNum, 0, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "ttlChangeOnWrite", tsTtlChangeOnWrite, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlFlushThreshold", tsTtlFlushThreshold, -1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3MigrateIntervalSec", tsS3MigrateIntervalSec, 600, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "s3MigrateEnabled", tsS3MigrateEnabled, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "timeseriesThreshold", tsTimeSeriesThreshold, 0, 2000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlBatchDropNum", tsTtlBatchDropNum, 0, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "ttlChangeOnWrite", tsTtlChangeOnWrite, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlFlushThreshold", tsTtlFlushThreshold, -1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3MigrateIntervalSec", tsS3MigrateIntervalSec, 600, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "s3MigrateEnabled", tsS3MigrateEnabled, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "timeseriesThreshold", tsTimeSeriesThreshold, 0, 2000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "udf", tsStartUdfd, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "udfdLdLibPath", tsUdfdLdLibPath, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "udf", tsStartUdfd, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "udfdLdLibPath", tsUdfdLdLibPath, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "streamAggCnt", tsStreamAggCnt, 2, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "streamAggCnt", tsStreamAggCnt, 2, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1800, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "concurrentCheckpoint", tsMaxConcurrentCheckpoint, 1, 10, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1800, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "concurrentCheckpoint", tsMaxConcurrentCheckpoint, 1, 10, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "curRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "ifAdtFse", tsIfAdtFse, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "compressor", tsCompressor, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "curRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "ifAdtFse", tsIfAdtFse, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "compressor", tsCompressor, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "resolveFQDNRetryTime", tsResolveFQDNRetryTime, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "resolveFQDNRetryTime", tsResolveFQDNRetryTime, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey[0], CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint[0], CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey[0], CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint[0], CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); // min free disk space used to check if the disk is full [50MB, 1GB] - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "minDiskFreeSize", tsMinDiskFreeSize, TFS_MIN_DISK_FREE_SIZE, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "minDiskFreeSize", tsMinDiskFreeSize, TFS_MIN_DISK_FREE_SIZE, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); // clang-format on @@ -1083,9 +1136,9 @@ int32_t taosSetSlowLogScope(char *pScopeStr, int32_t *pScope) { int32_t slowScope = 0; - char* scope = NULL; - char *tmp = NULL; - while((scope = strsep(&pScopeStr, "|")) != NULL){ + char *scope = NULL; + char *tmp = NULL; + while ((scope = strsep(&pScopeStr, "|")) != NULL) { taosMemoryFreeClear(tmp); tmp = taosStrdup(scope); if (tmp == NULL) { @@ -1142,13 +1195,13 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { (void)snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "firstEp"); - SEp firstEp = {0}; + SEp firstEp = {0}; TAOS_CHECK_RETURN(taosGetFqdnPortFromEp(strlen(pItem->str) == 0 ? defaultFirstEp : pItem->str, &firstEp)); (void)snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port); TAOS_CHECK_RETURN(cfgSetItem(pCfg, "firstEp", tsFirst, pItem->stype, true)); TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "secondEp"); - SEp secondEp = {0}; + SEp secondEp = {0}; TAOS_CHECK_RETURN(taosGetFqdnPortFromEp(strlen(pItem->str) == 0 ? defaultFirstEp : pItem->str, &secondEp)); (void)snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port); TAOS_CHECK_RETURN(cfgSetItem(pCfg, "secondEp", tsSecond, pItem->stype, true)); @@ -1308,7 +1361,6 @@ static int32_t taosSetSystemCfg(SConfig *pCfg) { taosSetCoreDump(tsEnableCoreFile); TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "assert"); - tsAssert = pItem->bval; // todo tsVersion = 30000000; @@ -1651,8 +1703,8 @@ static int32_t taosSetAllDebugFlag(SConfig *pCfg, int32_t flag); int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SConfig *pCfg = NULL; if (tsCfg == NULL) { @@ -1722,9 +1774,14 @@ int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char * SConfig *pCfg = NULL; TAOS_CHECK_RETURN(cfgInit(&pCfg)); - TAOS_CHECK_GOTO(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE), NULL, _exit); - TAOS_CHECK_GOTO(cfgAddInt32(pCfg, "debugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER), NULL, _exit); - TAOS_CHECK_GOTO(cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) ,NULL, _exit); + TAOS_CHECK_GOTO(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_LOCAL), NULL, + _exit); + TAOS_CHECK_GOTO( + cfgAddInt32(pCfg, "debugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL), NULL, + _exit); + TAOS_CHECK_GOTO( + cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL), NULL, + _exit); if ((code = taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) { (void)printf("failed to load cfg since %s\n", tstrerror(code)); @@ -1753,7 +1810,7 @@ _exit: static int32_t taosCheckGlobalCfg() { uint32_t ipv4 = 0; - int32_t code = taosGetIpv4FromFqdn(tsLocalFqdn, &ipv4); + int32_t code = taosGetIpv4FromFqdn(tsLocalFqdn, &ipv4); if (code) { uError("failed to get ip from fqdn:%s since %s, dnode can not be initialized", tsLocalFqdn, tstrerror(code)); TAOS_RETURN(TSDB_CODE_RPC_FQDN_ERROR); @@ -1858,7 +1915,7 @@ typedef struct { static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize, SConfigItem *pItem, bool isDebugflag) { int32_t code = TSDB_CODE_CFG_NOT_FOUND; - char *name = pItem->name; + char *name = pItem->name; for (int32_t d = 0; d < optionSize; ++d) { const char *optName = pOptions[d].optionName; if (strcasecmp(name, optName) != 0) continue; @@ -2053,8 +2110,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { } case 'f': { if (strcasecmp("fqdn", name) == 0) { - SConfigItem* pFqdnItem = cfgGetItem(pCfg, "fqdn"); - SConfigItem* pServerPortItem = cfgGetItem(pCfg, "serverPort"); + SConfigItem *pFqdnItem = cfgGetItem(pCfg, "fqdn"); + SConfigItem *pServerPortItem = cfgGetItem(pCfg, "serverPort"); SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp"); if (pFqdnItem == NULL || pServerPortItem == NULL || pFirstEpItem == NULL) { uError("failed to get fqdn or serverPort or firstEp from cfg"); @@ -2069,7 +2126,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { char defaultFirstEp[TSDB_EP_LEN] = {0}; (void)snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); - SEp firstEp = {0}; + SEp firstEp = {0}; TAOS_CHECK_GOTO( taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp), &lino, _out); @@ -2109,8 +2166,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { } case 'l': { if (strcasecmp("locale", name) == 0) { - SConfigItem* pLocaleItem = cfgGetItem(pCfg, "locale"); - SConfigItem* pCharsetItem = cfgGetItem(pCfg, "charset"); + SConfigItem *pLocaleItem = cfgGetItem(pCfg, "locale"); + SConfigItem *pCharsetItem = cfgGetItem(pCfg, "charset"); if (pLocaleItem == NULL || pCharsetItem == NULL) { uError("failed to get locale or charset from cfg"); code = TSDB_CODE_CFG_NOT_FOUND; @@ -2183,7 +2240,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { char defaultFirstEp[TSDB_EP_LEN] = {0}; (void)snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); - SEp firstEp = {0}; + SEp firstEp = {0}; TAOS_CHECK_GOTO( taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp), &lino, _out); @@ -2231,7 +2288,6 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { }; static OptionNameAndVar options[] = {{"asyncLog", &tsAsyncLog}, - {"assert", &tsAssert}, {"compressMsgSize", &tsCompressMsgSize}, {"countAlwaysReturnValue", &tsCountAlwaysReturnValue}, {"crashReporting", &tsEnableCrashReport}, @@ -2314,7 +2370,7 @@ int32_t taosSetGlobalDebugFlag(int32_t flag) { return taosSetAllDebugFlag(tsCfg, // NOTE: set all command does not change the tmrDebugFlag static int32_t taosSetAllDebugFlag(SConfig *pCfg, int32_t flag) { if (flag < 0) TAOS_RETURN(TSDB_CODE_INVALID_PARA); - if (flag == 0) TAOS_RETURN(TSDB_CODE_SUCCESS); // just ignore + if (flag == 0) TAOS_RETURN(TSDB_CODE_SUCCESS); // just ignore SArray *noNeedToSetVars = NULL; SConfigItem *pItem = NULL; diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index d6852b0566..afe8bf7ef9 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -533,13 +533,13 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { TAOS_RETURN(TSDB_CODE_SUCCESS); } -int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope, int8_t dynScope) { +int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope, int8_t dynScope, int8_t category) { SConfigItem item = {.dtype = CFG_DTYPE_BOOL, .bval = defaultVal, .scope = scope, .dynScope = dynScope}; return cfgAddItem(pCfg, &item, name); } int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, - int8_t dynScope) { + int8_t dynScope, int8_t category) { if (defaultVal < minval || defaultVal > maxval) { TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } @@ -554,7 +554,7 @@ int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t } int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, - int8_t dynScope) { + int8_t dynScope, int8_t category) { if (defaultVal < minval || defaultVal > maxval) { TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } @@ -569,7 +569,7 @@ int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t } int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope, - int8_t dynScope) { + int8_t dynScope, int8_t category) { if (defaultVal < minval || defaultVal > maxval) { TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } @@ -583,7 +583,8 @@ int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float min return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) { +int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope, + int8_t category) { SConfigItem item = {.dtype = CFG_DTYPE_STRING, .scope = scope, .dynScope = dynScope}; item.str = taosStrdup(defaultVal); if (item.str == NULL) { @@ -592,25 +593,29 @@ int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, in return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) { +int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope, + int8_t category) { SConfigItem item = {.dtype = CFG_DTYPE_DIR, .scope = scope, .dynScope = dynScope}; TAOS_CHECK_RETURN(cfgCheckAndSetDir(&item, defaultVal)); return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) { +int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope, + int8_t category) { SConfigItem item = {.dtype = CFG_DTYPE_LOCALE, .scope = scope, .dynScope = dynScope}; TAOS_CHECK_RETURN(cfgCheckAndSetConf(&item, defaultVal)); return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) { +int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope, + int8_t category) { SConfigItem item = {.dtype = CFG_DTYPE_CHARSET, .scope = scope, .dynScope = dynScope}; TAOS_CHECK_RETURN(cfgCheckAndSetConf(&item, defaultVal)); return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) { +int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope, + int8_t category) { SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE, .scope = scope, .dynScope = dynScope}; TAOS_CHECK_RETURN(cfgCheckAndSetConf(&item, defaultVal)); return cfgAddItem(pCfg, &item, name); diff --git a/source/util/test/cfgTest.cpp b/source/util/test/cfgTest.cpp index 33ababef10..a5812d375b 100644 --- a/source/util/test/cfgTest.cpp +++ b/source/util/test/cfgTest.cpp @@ -57,12 +57,12 @@ TEST_F(CfgTest, 02_Basic) { ASSERT_EQ(code, TSDB_CODE_SUCCESS); ASSERT_NE(pConfig, nullptr); - EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, 0, 0), 0); - EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 1, 0, 16, 0, 0), 0); - EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16, 0, 0), 0); - EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16, 0, 0), 0); - EXPECT_EQ(cfgAddString(pConfig, "test_string", "4", 0, 0), 0); - EXPECT_EQ(cfgAddDir(pConfig, "test_dir", TD_TMP_DIR_PATH, 0, 0), 0); + EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, 0, 0, 0), 0); + EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 1, 0, 16, 0, 0, 0), 0); + EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16, 0, 0, 0), 0); + EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16, 0, 0, 0), 0); + EXPECT_EQ(cfgAddString(pConfig, "test_string", "4", 0, 0, 0), 0); + EXPECT_EQ(cfgAddDir(pConfig, "test_dir", TD_TMP_DIR_PATH, 0, 0, 0), 0); EXPECT_EQ(cfgGetSize(pConfig), 6); From f68bbfa448d354efb26cdb655425b1b5d321e1e9 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 24 Oct 2024 14:37:50 +0800 Subject: [PATCH 02/99] drop conf assert --- source/common/src/tglobal.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 9a63ab5013..f92e4b5f04 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1329,7 +1329,7 @@ static int32_t taosSetSystemCfg(SConfig *pCfg) { SConfigItem *pItem = NULL; TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "timezone"); - if (0 == strlen(pItem->str)) { + if (0 == strlen(pItem->str)) { uError("timezone is not set"); } else { TAOS_CHECK_RETURN(osSetTimezone(pItem->str)); @@ -1360,8 +1360,6 @@ static int32_t taosSetSystemCfg(SConfig *pCfg) { tsEnableCoreFile = pItem->bval; taosSetCoreDump(tsEnableCoreFile); - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "assert"); - // todo tsVersion = 30000000; From 9f6504cdf4255b47af106814a908c8837d822fde Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Fri, 1 Nov 2024 10:24:26 +0800 Subject: [PATCH 03/99] Add category to config item and rename array to loaclArray. --- include/util/tconfig.h | 1 + source/util/src/tconfig.c | 58 +++++++++++++++++++++------------------ 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/include/util/tconfig.h b/include/util/tconfig.h index d687d59ac2..6318478509 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -83,6 +83,7 @@ typedef struct SConfigItem { ECfgDataType dtype; int8_t scope; int8_t dynScope; + int8_t category; char *name; union { bool bval; diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index afe8bf7ef9..5a1dcf13ae 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -14,9 +14,9 @@ */ #define _DEFAULT_SOURCE -#include "tconfig.h" #include "cJSON.h" #include "taoserror.h" +#include "tconfig.h" #include "tenv.h" #include "tglobal.h" #include "tgrant.h" @@ -30,7 +30,7 @@ struct SConfig { ECfgSrcType stype; - SArray *array; + SArray *localArray; TdThreadMutex lock; }; @@ -48,8 +48,8 @@ int32_t cfgInit(SConfig **ppCfg) { TAOS_RETURN(terrno); } - pCfg->array = taosArrayInit(32, sizeof(SConfigItem)); - if (pCfg->array == NULL) { + pCfg->localArray = taosArrayInit(32, sizeof(SConfigItem)); + if (pCfg->localArray == NULL) { taosMemoryFree(pCfg); TAOS_RETURN(terrno); } @@ -105,19 +105,19 @@ void cfgCleanup(SConfig *pCfg) { return; } - int32_t size = taosArrayGetSize(pCfg->array); + int32_t size = taosArrayGetSize(pCfg->localArray); for (int32_t i = 0; i < size; ++i) { - SConfigItem *pItem = taosArrayGet(pCfg->array, i); + SConfigItem *pItem = taosArrayGet(pCfg->localArray, i); cfgItemFreeVal(pItem); taosMemoryFreeClear(pItem->name); } - taosArrayDestroy(pCfg->array); + taosArrayDestroy(pCfg->localArray); (void)taosThreadMutexDestroy(&pCfg->lock); taosMemoryFree(pCfg); } -int32_t cfgGetSize(SConfig *pCfg) { return taosArrayGetSize(pCfg->array); } +int32_t cfgGetSize(SConfig *pCfg) { return taosArrayGetSize(pCfg->localArray); } static int32_t cfgCheckAndSetConf(SConfigItem *pItem, const char *conf) { cfgItemFreeVal(pItem); @@ -390,9 +390,9 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName) { if (pCfg == NULL) return NULL; - int32_t size = taosArrayGetSize(pCfg->array); + int32_t size = taosArrayGetSize(pCfg->localArray); for (int32_t i = 0; i < size; ++i) { - SConfigItem *pItem = taosArrayGet(pCfg->array, i); + SConfigItem *pItem = taosArrayGet(pCfg->localArray, i); if (strcasecmp(pItem->name, pName) == 0) { return pItem; } @@ -508,9 +508,9 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { TAOS_RETURN(terrno); } - int32_t size = taosArrayGetSize(pCfg->array); + int32_t size = taosArrayGetSize(pCfg->localArray); for (int32_t i = 0; i < size; ++i) { - SConfigItem *existItem = taosArrayGet(pCfg->array, i); + SConfigItem *existItem = taosArrayGet(pCfg->localArray, i); if (existItem != NULL && strcmp(existItem->name, pItem->name) == 0) { taosMemoryFree(pItem->name); TAOS_RETURN(TSDB_CODE_INVALID_CFG); @@ -521,7 +521,7 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; (void)strntolower(lowcaseName, name, TMIN(CFG_NAME_MAX_LEN, len)); - if (taosArrayPush(pCfg->array, pItem) == NULL) { + if (taosArrayPush(pCfg->localArray, pItem) == NULL) { if (pItem->dtype == CFG_DTYPE_STRING) { taosMemoryFree(pItem->str); } @@ -534,7 +534,8 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { } int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope, int8_t dynScope, int8_t category) { - SConfigItem item = {.dtype = CFG_DTYPE_BOOL, .bval = defaultVal, .scope = scope, .dynScope = dynScope}; + SConfigItem item = { + .dtype = CFG_DTYPE_BOOL, .bval = defaultVal, .scope = scope, .dynScope = dynScope, .category = category}; return cfgAddItem(pCfg, &item, name); } @@ -549,7 +550,8 @@ int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t .imin = minval, .imax = maxval, .scope = scope, - .dynScope = dynScope}; + .dynScope = dynScope, + .category = category}; return cfgAddItem(pCfg, &item, name); } @@ -564,7 +566,8 @@ int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t .imin = minval, .imax = maxval, .scope = scope, - .dynScope = dynScope}; + .dynScope = dynScope, + .category = category}; return cfgAddItem(pCfg, &item, name); } @@ -579,13 +582,14 @@ int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float min .fmin = minval, .fmax = maxval, .scope = scope, - .dynScope = dynScope}; + .dynScope = dynScope, + .category = category}; return cfgAddItem(pCfg, &item, name); } int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope, int8_t category) { - SConfigItem item = {.dtype = CFG_DTYPE_STRING, .scope = scope, .dynScope = dynScope}; + SConfigItem item = {.dtype = CFG_DTYPE_STRING, .scope = scope, .dynScope = dynScope, .category = category}; item.str = taosStrdup(defaultVal); if (item.str == NULL) { TAOS_RETURN(terrno); @@ -595,28 +599,28 @@ int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, in int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope, int8_t category) { - SConfigItem item = {.dtype = CFG_DTYPE_DIR, .scope = scope, .dynScope = dynScope}; + SConfigItem item = {.dtype = CFG_DTYPE_DIR, .scope = scope, .dynScope = dynScope, .category = category}; TAOS_CHECK_RETURN(cfgCheckAndSetDir(&item, defaultVal)); return cfgAddItem(pCfg, &item, name); } int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope, int8_t category) { - SConfigItem item = {.dtype = CFG_DTYPE_LOCALE, .scope = scope, .dynScope = dynScope}; + SConfigItem item = {.dtype = CFG_DTYPE_LOCALE, .scope = scope, .dynScope = dynScope, .category = category}; TAOS_CHECK_RETURN(cfgCheckAndSetConf(&item, defaultVal)); return cfgAddItem(pCfg, &item, name); } int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope, int8_t category) { - SConfigItem item = {.dtype = CFG_DTYPE_CHARSET, .scope = scope, .dynScope = dynScope}; + SConfigItem item = {.dtype = CFG_DTYPE_CHARSET, .scope = scope, .dynScope = dynScope, .category = category}; TAOS_CHECK_RETURN(cfgCheckAndSetConf(&item, defaultVal)); return cfgAddItem(pCfg, &item, name); } int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope, int8_t category) { - SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE, .scope = scope, .dynScope = dynScope}; + SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE, .scope = scope, .dynScope = dynScope, .category = category}; TAOS_CHECK_RETURN(cfgCheckAndSetConf(&item, defaultVal)); return cfgAddItem(pCfg, &item, name); } @@ -751,9 +755,9 @@ void cfgDumpCfgS3(SConfig *pCfg, bool tsc, bool dump) { char src[CFG_SRC_PRINT_LEN + 1] = {0}; char name[CFG_NAME_PRINT_LEN + 1] = {0}; - int32_t size = taosArrayGetSize(pCfg->array); + int32_t size = taosArrayGetSize(pCfg->localArray); for (int32_t i = 0; i < size; ++i) { - SConfigItem *pItem = taosArrayGet(pCfg->array, i); + SConfigItem *pItem = taosArrayGet(pCfg->localArray, i); if (tsc && pItem->scope == CFG_SCOPE_SERVER) continue; if (dump && strcmp(pItem->name, "scriptDir") == 0) continue; if (dump && strncmp(pItem->name, "s3", 2) != 0) continue; @@ -834,9 +838,9 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { char src[CFG_SRC_PRINT_LEN + 1] = {0}; char name[CFG_NAME_PRINT_LEN + 1] = {0}; - int32_t size = taosArrayGetSize(pCfg->array); + int32_t size = taosArrayGetSize(pCfg->localArray); for (int32_t i = 0; i < size; ++i) { - SConfigItem *pItem = taosArrayGet(pCfg->array, i); + SConfigItem *pItem = taosArrayGet(pCfg->localArray, i); if (tsc && pItem->scope == CFG_SCOPE_SERVER) continue; if (dump && strcmp(pItem->name, "scriptDir") == 0) continue; tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN); @@ -1490,7 +1494,7 @@ int32_t cfgCreateIter(SConfig *pConf, SConfigIter **ppIter) { SConfigItem *cfgNextIter(SConfigIter *pIter) { if (pIter->index < cfgGetSize(pIter->pConf)) { - return taosArrayGet(pIter->pConf->array, pIter->index++); + return taosArrayGet(pIter->pConf->localArray, pIter->index++); } return NULL; From 5cd23d8f6cbad6a704523725f507eaf37faab257 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Fri, 1 Nov 2024 15:20:31 +0800 Subject: [PATCH 04/99] Split array in pCfg to global&local array. --- source/util/src/tconfig.c | 75 +++++++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 23 deletions(-) diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 5a1dcf13ae..0ec536a21c 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -31,6 +31,7 @@ struct SConfig { ECfgSrcType stype; SArray *localArray; + SArray *globalArray; TdThreadMutex lock; }; @@ -48,11 +49,16 @@ int32_t cfgInit(SConfig **ppCfg) { TAOS_RETURN(terrno); } - pCfg->localArray = taosArrayInit(32, sizeof(SConfigItem)); + pCfg->localArray = taosArrayInit(64, sizeof(SConfigItem)); if (pCfg->localArray == NULL) { taosMemoryFree(pCfg); TAOS_RETURN(terrno); } + pCfg->globalArray = taosArrayInit(64, sizeof(SConfigItem)); + if (pCfg->globalArray == NULL) { + taosMemoryFree(pCfg); + TAOS_RETURN(terrno); + } TAOS_CHECK_RETURN(taosThreadMutexInit(&pCfg->lock, NULL)); *ppCfg = pCfg; @@ -112,12 +118,22 @@ void cfgCleanup(SConfig *pCfg) { taosMemoryFreeClear(pItem->name); } + size = taosArrayGetSize(pCfg->globalArray); + for (int32_t i = 0; i < size; ++i) { + SConfigItem *pItem = taosArrayGet(pCfg->globalArray, i); + cfgItemFreeVal(pItem); + taosMemoryFreeClear(pItem->name); + } + taosArrayDestroy(pCfg->localArray); + taosArrayDestroy(pCfg->globalArray); (void)taosThreadMutexDestroy(&pCfg->lock); taosMemoryFree(pCfg); } -int32_t cfgGetSize(SConfig *pCfg) { return taosArrayGetSize(pCfg->localArray); } +int32_t cfgGetSize(SConfig *pCfg) { return taosArrayGetSize(pCfg->localArray) + taosArrayGetSize(pCfg->globalArray); } +int32_t cfgGetLocalSize(SConfig *pCfg) { return taosArrayGetSize(pCfg->localArray); } +int32_t cfgGetGlobalSize(SConfig *pCfg) { return taosArrayGetSize(pCfg->globalArray); } static int32_t cfgCheckAndSetConf(SConfigItem *pItem, const char *conf) { cfgItemFreeVal(pItem); @@ -397,6 +413,13 @@ SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName) { return pItem; } } + size = taosArrayGetSize(pCfg->globalArray); + for (int32_t i = 0; i < size; ++i) { + SConfigItem *pItem = taosArrayGet(pCfg->globalArray, i); + if (strcasecmp(pItem->name, pName) == 0) { + return pItem; + } + } return NULL; } @@ -502,15 +525,18 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p } static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { + SArray *array = pCfg->globalArray; + if (pItem->category == CFG_CATEGORY_LOCAL) array = pCfg->localArray; + pItem->stype = CFG_STYPE_DEFAULT; pItem->name = taosStrdup(name); if (pItem->name == NULL) { TAOS_RETURN(terrno); } - int32_t size = taosArrayGetSize(pCfg->localArray); + int32_t size = taosArrayGetSize(array); for (int32_t i = 0; i < size; ++i) { - SConfigItem *existItem = taosArrayGet(pCfg->localArray, i); + SConfigItem *existItem = taosArrayGet(array, i); if (existItem != NULL && strcmp(existItem->name, pItem->name) == 0) { taosMemoryFree(pItem->name); TAOS_RETURN(TSDB_CODE_INVALID_CFG); @@ -521,7 +547,7 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; (void)strntolower(lowcaseName, name, TMIN(CFG_NAME_MAX_LEN, len)); - if (taosArrayPush(pCfg->localArray, pItem) == NULL) { + if (taosArrayPush(array, pItem) == NULL) { if (pItem->dtype == CFG_DTYPE_STRING) { taosMemoryFree(pItem->str); } @@ -741,23 +767,12 @@ int32_t cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t TAOS_RETURN(TSDB_CODE_SUCCESS); } -void cfgDumpCfgS3(SConfig *pCfg, bool tsc, bool dump) { - if (dump) { - (void)printf(" s3 config"); - (void)printf("\n"); - (void)printf("================================================================="); - (void)printf("\n"); - } else { - uInfo(" s3 config"); - uInfo("================================================================="); - } - - char src[CFG_SRC_PRINT_LEN + 1] = {0}; - char name[CFG_NAME_PRINT_LEN + 1] = {0}; - - int32_t size = taosArrayGetSize(pCfg->localArray); +void cfgDumpCfgArrayS3(SArray *array, bool tsc, bool dump) { + char src[CFG_SRC_PRINT_LEN + 1] = {0}; + char name[CFG_NAME_PRINT_LEN + 1] = {0}; + int32_t size = taosArrayGetSize(array); for (int32_t i = 0; i < size; ++i) { - SConfigItem *pItem = taosArrayGet(pCfg->localArray, i); + SConfigItem *pItem = taosArrayGet(array, i); if (tsc && pItem->scope == CFG_SCOPE_SERVER) continue; if (dump && strcmp(pItem->name, "scriptDir") == 0) continue; if (dump && strncmp(pItem->name, "s3", 2) != 0) continue; @@ -816,7 +831,20 @@ void cfgDumpCfgS3(SConfig *pCfg, bool tsc, bool dump) { break; } } +} +void cfgDumpCfgS3(SConfig *pCfg, bool tsc, bool dump) { + if (dump) { + (void)printf(" s3 config"); + (void)printf("\n"); + (void)printf("================================================================="); + (void)printf("\n"); + } else { + uInfo(" s3 config"); + uInfo("================================================================="); + } + cfgDumpCfgArrayS3(pCfg->localArray, tsc, dump); + cfgDumpCfgArrayS3(pCfg->globalArray, tsc, dump); if (dump) { (void)printf("=================================================================\n"); } else { @@ -1493,10 +1521,11 @@ int32_t cfgCreateIter(SConfig *pConf, SConfigIter **ppIter) { } SConfigItem *cfgNextIter(SConfigIter *pIter) { - if (pIter->index < cfgGetSize(pIter->pConf)) { + if (pIter->index < cfgGetGlobalSize(pIter->pConf)) { return taosArrayGet(pIter->pConf->localArray, pIter->index++); + } else if (pIter->index < cfgGetGlobalSize(pIter->pConf) + cfgGetLocalSize(pIter->pConf)) { + return taosArrayGet(pIter->pConf->globalArray, pIter->index++ - cfgGetGlobalSize(pIter->pConf)); } - return NULL; } From bfceaae124e7ad939af8f9d93675a8554d74a45b Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Fri, 1 Nov 2024 16:29:47 +0800 Subject: [PATCH 05/99] fix CFG_DYN_ENT_READONLY declare error --- include/util/tconfig.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 6318478509..b5dcc69d16 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -74,7 +74,7 @@ typedef enum { CFG_DYN_ENT_SERVER = CFG_DYN_NONE, CFG_DYN_ENT_CLIENT = CFG_DYN_NONE, CFG_DYN_ENT_BOTH = CFG_DYN_NONE, - CFG_DYN_READONLY = CFG_DYN_NONE, + CFG_DYN_ENT_READONLY = CFG_DYN_NONE, #endif } ECfgDynType; From 07de144000c24072c936f5f946721227141061a3 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 4 Nov 2024 16:16:24 +0800 Subject: [PATCH 06/99] serialize and persist local config --- include/common/tglobal.h | 4 + include/common/tmsg.h | 21 +++-- include/dnode/mnode/mnode.h | 12 ++- include/util/tconfig.h | 3 +- source/common/src/tglobal.c | 98 +++++++++++++++++++++++- source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 14 ++++ source/util/src/tconfig.c | 3 + 7 files changed, 139 insertions(+), 16 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 41fb692e42..c6757cd735 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -282,6 +282,10 @@ void taosLocalCfgForbiddenToChange(char *name, bool *forbidden); int8_t taosGranted(int8_t type); int32_t taosSetSlowLogScope(char *pScopeStr, int32_t *pScope); +int32_t persistLocalConfig(SConfig *pCfg); +int32_t localConfigSerialize(SArray *array, char **serialized); +int32_t tSerializeSConfigArray(void *buf, int32_t bufLen, SArray *array); +int32_t tDeserializeSConfigArray(void *buf, int32_t bufLen, SArray *pReq); #ifdef __cplusplus } #endif diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 8366837b29..e8441b285b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -421,7 +421,7 @@ typedef enum ENodeType { // physical plan node QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN = 1100, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, - QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, // INACTIVE + QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, // INACTIVE QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, @@ -435,7 +435,7 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_SORT, QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT, QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, - QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, // INACTIVE + QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, // INACTIVE QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL, @@ -983,7 +983,6 @@ typedef struct SEpSet { SEp eps[TSDB_MAX_REPLICA]; } SEpSet; - int32_t tEncodeSEpSet(SEncoder* pEncoder, const SEpSet* pEp); int32_t tDecodeSEpSet(SDecoder* pDecoder, SEpSet* pEp); int32_t taosEncodeSEpSet(void** buf, const SEpSet* pEp); @@ -4115,20 +4114,20 @@ typedef struct { SArray* blockTbName; SArray* blockSchema; - union{ - struct{ - int64_t sleepTime; + union { + struct { + int64_t sleepTime; }; - struct{ - int32_t createTableNum; - SArray* createTableLen; - SArray* createTableReq; + struct { + int32_t createTableNum; + SArray* createTableLen; + SArray* createTableReq; }; }; } SMqDataRsp; -int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pObj); +int32_t tEncodeMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pObj); int32_t tDecodeMqDataRsp(SDecoder* pDecoder, SMqDataRsp* pRsp); void tDeleteMqDataRsp(SMqDataRsp* pRsp); diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 4abf8c27fd..9bdb6eac37 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -17,10 +17,10 @@ #define _TD_MND_H_ #include "monitor.h" +#include "sync.h" #include "tmsg.h" #include "tmsgcb.h" #include "trpc.h" -#include "sync.h" #ifdef __cplusplus extern "C" { @@ -41,6 +41,12 @@ typedef struct { int64_t lastIndex; } SMnodeOpt; +typedef struct { + int32_t version; + SArray *pArray; + TdThreadMutex mutex; +} SMnodeConfig; + /* ------------------------ SMnode ------------------------ */ /** * @brief Open a mnode. @@ -73,7 +79,7 @@ int32_t mndStart(SMnode *pMnode); */ void mndStop(SMnode *pMnode); -int32_t mndIsCatchUp(SMnode *pMnode); +int32_t mndIsCatchUp(SMnode *pMnode); ESyncRole mndGetRole(SMnode *pMnode); int64_t mndGetTerm(SMnode *pMnode); @@ -109,7 +115,7 @@ int64_t mndGetRoleTimeMs(SMnode *pMnode); * @param pMsg The request msg. * @return int32_t 0 for success, -1 for failure. */ -int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo* pQueueInfo); +int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo *pQueueInfo); int32_t mndProcessSyncMsg(SRpcMsg *pMsg); int32_t mndPreProcessQueryMsg(SRpcMsg *pMsg); void mndPostProcessQueryMsg(SRpcMsg *pMsg); diff --git a/include/util/tconfig.h b/include/util/tconfig.h index b5dcc69d16..197bf04c3c 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -148,7 +148,8 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump); void cfgDumpCfgS3(SConfig *pCfg, bool tsc, bool dump); int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl); - +SArray *cfgGetLocalCfg(SConfig *pCfg); +SArray *cfgGetGlobalCfg(SConfig *pCfg); #ifdef __cplusplus } #endif diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 0db7f3dd93..a0cff8dafc 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE +#include "cJSON.h" #include "defines.h" #include "os.h" #include "tconfig.h" @@ -27,6 +28,9 @@ #include "cus_name.h" #endif +#define CONFIG_PATH_LEN (TSDB_FILENAME_LEN + 12) +#define CONFIG_FILE_LEN (CONFIG_PATH_LEN + 32) + // GRANT_CFG_DECLARE; SConfig *tsCfg = NULL; @@ -1927,7 +1931,11 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile TAOS_CHECK_GOTO(taosSetAllDebugFlag(tsCfg, pItem->i32), &lino, _exit); cfgDumpCfg(tsCfg, tsc, false); - + if (!tsc) { + if ((code = persistLocalConfig(tsCfg)) != 0) { + goto _exit; + } + } TAOS_CHECK_GOTO(taosCheckGlobalCfg(), &lino, _exit); _exit: @@ -2473,3 +2481,91 @@ int8_t taosGranted(int8_t type) { } return 0; } + +int32_t persistLocalConfig(SConfig *pCfg) { + // TODO: just tmp ,refactor later + int32_t code = 0; + char *buffer = NULL; + TdFilePtr pFile = NULL; + char filepath[CONFIG_FILE_LEN] = {0}; + char filename[CONFIG_FILE_LEN] = {0}; + snprintf(filepath, sizeof(filepath), "%s%sconfig", tsDataDir, TD_DIRSEP); + snprintf(filename, sizeof(filename), "%s%sconfig%slocal.json", tsDataDir, TD_DIRSEP, TD_DIRSEP); + + // TODO(beryl) need to check if the file is existed + if (taosMkDir(filepath) != 0) { + code = TAOS_SYSTEM_ERROR(errno); + uError("failed to create dir:%s since %s", filepath, tstrerror(code)); + TAOS_RETURN(code); + } + + TdFilePtr pConfigFile = + taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); + + if (pConfigFile == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + uError("failed to open file:%s since %s", filename, tstrerror(code)); + TAOS_RETURN(code); + } + char *serialized = NULL; + code = localConfigSerialize(cfgGetLocalCfg(pCfg), &serialized); + if (code != TSDB_CODE_SUCCESS) { + uError("failed to serialize local config since %s", tstrerror(code)); + TAOS_RETURN(code); + } + taosWriteFile(pConfigFile, serialized, strlen(serialized)); + return TSDB_CODE_SUCCESS; +} + +int32_t localConfigSerialize(SArray *array, char **serialized) { + char buf[30]; + cJSON *json = cJSON_CreateObject(); + if (json == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int sz = taosArrayGetSize(array); + + cJSON *cField = cJSON_CreateArray(); + if (array == NULL) { + cJSON_Delete(json); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } + // cjson only support int32_t or double + // string are used to prohibit the loss of precision + for (int i = 0; i < sz; i++) { + SConfigItem *item = (SConfigItem *)taosArrayGet(array, i); + switch (item->dtype) { + { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + cJSON_AddBoolToObject(cField, item->name, item->bval); + break; + case CFG_DTYPE_INT32: + cJSON_AddNumberToObject(cField, item->name, item->i32); + break; + case CFG_DTYPE_INT64: + (void)sprintf(buf, "%" PRId64, item->i64); + cJSON_AddStringToObject(cField, item->name, buf); + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + (void)sprintf(buf, "%f", item->fval); + cJSON_AddStringToObject(cField, item->name, buf); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + cJSON_AddStringToObject(cField, item->name, item->str); + break; + } + } + } + cJSON_AddItemToObject(json, "pArray", cField); + *serialized = cJSON_Print(json); + cJSON_Delete(json); + return TSDB_CODE_SUCCESS; +} \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index b1b7a90db8..cad9154643 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -45,6 +45,19 @@ static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) { return code; } +static void mmBuildConfigForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInput, SMnodeOpt *pOption) { + pOption->deploy = true; + pOption->msgCb = pMgmt->msgCb; + pOption->dnodeId = pMgmt->pData->dnodeId; + pOption->selfIndex = 0; + pOption->numOfReplicas = 1; + pOption->numOfTotalReplicas = 1; + pOption->replicas[0].id = 1; + pOption->replicas[0].port = tsServerPort; + tstrncpy(pOption->replicas[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN); + pOption->lastIndex = SYNC_INDEX_INVALID; +} + static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInput, SMnodeOpt *pOption) { pOption->deploy = true; pOption->msgCb = pMgmt->msgCb; @@ -120,6 +133,7 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { dInfo("mnode start to deploy"); pMgmt->pData->dnodeId = 1; mmBuildOptionForDeploy(pMgmt, pInput, &option); + } else { dInfo("mnode start to open"); mmBuildOptionForOpen(pMgmt, &option); diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 0ec536a21c..3bac376242 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -1536,3 +1536,6 @@ void cfgDestroyIter(SConfigIter *pIter) { taosMemoryFree(pIter); } + +SArray *cfgGetLocalCfg(SConfig *pCfg) { return pCfg->localArray; } +SArray *cfgGetGlobalCfg(SConfig *pCfg) { return pCfg->globalArray; } \ No newline at end of file From aeafb49673740fb4260cfae467eb5cd8d4d783be Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 5 Nov 2024 10:04:36 +0800 Subject: [PATCH 07/99] modify local.json format --- source/common/src/tglobal.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index a0cff8dafc..7bf3cbe8d5 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -2526,7 +2526,7 @@ int32_t localConfigSerialize(SArray *array, char **serialized) { int sz = taosArrayGetSize(array); - cJSON *cField = cJSON_CreateArray(); + cJSON *cField = cJSON_CreateObject(); if (array == NULL) { cJSON_Delete(json); TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); @@ -2564,7 +2564,7 @@ int32_t localConfigSerialize(SArray *array, char **serialized) { } } } - cJSON_AddItemToObject(json, "pArray", cField); + cJSON_AddItemToObject(json, "configs", cField); *serialized = cJSON_Print(json); cJSON_Delete(json); return TSDB_CODE_SUCCESS; From 76480feed2a56c5af13354ecd32f8aa0c35e916d Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 7 Nov 2024 10:44:55 +0800 Subject: [PATCH 08/99] persist global.cfg --- include/common/tglobal.h | 3 +- include/util/taoserror.h | 2 +- source/common/src/tglobal.c | 149 ++++++++++++++++++----- source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 15 +-- 4 files changed, 124 insertions(+), 45 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index c6757cd735..86736295c9 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -282,7 +282,8 @@ void taosLocalCfgForbiddenToChange(char *name, bool *forbidden); int8_t taosGranted(int8_t type); int32_t taosSetSlowLogScope(char *pScopeStr, int32_t *pScope); -int32_t persistLocalConfig(SConfig *pCfg); +int32_t persistGlobalConfig(const char *path, int32_t version); +int32_t persistLocalConfig(const char *path); int32_t localConfigSerialize(SArray *array, char **serialized); int32_t tSerializeSConfigArray(void *buf, int32_t bufLen, SArray *array); int32_t tDeserializeSConfigArray(void *buf, int32_t bufLen, SArray *pReq); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 2a67a3cae1..7c72feea8b 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -577,7 +577,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_TDB_INVALID_CREATE_TB_MSG TAOS_DEF_ERROR_CODE(0, 0x060E) #define TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM TAOS_DEF_ERROR_CODE(0, 0x060F) #define TSDB_CODE_TDB_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0610) -#define TSDB_CODE_TDB_TABLE_RECONFIGURE TAOS_DEF_ERROR_CODE(0, 0x0611) +#define TSDB_CODE_TDB_TABLE_RECONFIGURE TAOS_DEF_ERROR_CODE(0, 0x0611) #define TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO TAOS_DEF_ERROR_CODE(0, 0x0612) #define TSDB_CODE_TDB_NO_AVAIL_DISK TAOS_DEF_ERROR_CODE(0, 0x0613) #define TSDB_CODE_TDB_MESSED_MSG TAOS_DEF_ERROR_CODE(0, 0x0614) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 7bf3cbe8d5..02b9b44e40 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1932,7 +1932,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile cfgDumpCfg(tsCfg, tsc, false); if (!tsc) { - if ((code = persistLocalConfig(tsCfg)) != 0) { + if ((code = persistLocalConfig(tsDataDir)) != 0) { goto _exit; } } @@ -2482,38 +2482,57 @@ int8_t taosGranted(int8_t type) { return 0; } -int32_t persistLocalConfig(SConfig *pCfg) { - // TODO: just tmp ,refactor later - int32_t code = 0; - char *buffer = NULL; - TdFilePtr pFile = NULL; - char filepath[CONFIG_FILE_LEN] = {0}; - char filename[CONFIG_FILE_LEN] = {0}; - snprintf(filepath, sizeof(filepath), "%s%sconfig", tsDataDir, TD_DIRSEP); - snprintf(filename, sizeof(filename), "%s%sconfig%slocal.json", tsDataDir, TD_DIRSEP, TD_DIRSEP); +int32_t globalConfigSerialize(int32_t version, SArray *array, char **serialized) { + char buf[30]; + cJSON *json = cJSON_CreateObject(); + if (json == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + cJSON_AddNumberToObject(json, "version", version); + int sz = taosArrayGetSize(array); - // TODO(beryl) need to check if the file is existed - if (taosMkDir(filepath) != 0) { - code = TAOS_SYSTEM_ERROR(errno); - uError("failed to create dir:%s since %s", filepath, tstrerror(code)); - TAOS_RETURN(code); + cJSON *cField = cJSON_CreateObject(); + if (cField == NULL) { + cJSON_Delete(json); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } - TdFilePtr pConfigFile = - taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); - - if (pConfigFile == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - uError("failed to open file:%s since %s", filename, tstrerror(code)); - TAOS_RETURN(code); + // cjson only support int32_t or double + // string are used to prohibit the loss of precision + for (int i = 0; i < sz; i++) { + SConfigItem *item = (SConfigItem *)taosArrayGet(array, i); + switch (item->dtype) { + { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + cJSON_AddBoolToObject(cField, item->name, item->bval); + break; + case CFG_DTYPE_INT32: + cJSON_AddNumberToObject(cField, item->name, item->i32); + break; + case CFG_DTYPE_INT64: + (void)sprintf(buf, "%" PRId64, item->i64); + cJSON_AddStringToObject(cField, item->name, buf); + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + (void)sprintf(buf, "%f", item->fval); + cJSON_AddStringToObject(cField, item->name, buf); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + cJSON_AddStringToObject(cField, item->name, item->str); + break; + } + } } - char *serialized = NULL; - code = localConfigSerialize(cfgGetLocalCfg(pCfg), &serialized); - if (code != TSDB_CODE_SUCCESS) { - uError("failed to serialize local config since %s", tstrerror(code)); - TAOS_RETURN(code); - } - taosWriteFile(pConfigFile, serialized, strlen(serialized)); + cJSON_AddItemToObject(json, "configs", cField); + *serialized = cJSON_Print(json); + cJSON_Delete(json); return TSDB_CODE_SUCCESS; } @@ -2568,4 +2587,74 @@ int32_t localConfigSerialize(SArray *array, char **serialized) { *serialized = cJSON_Print(json); cJSON_Delete(json); return TSDB_CODE_SUCCESS; -} \ No newline at end of file +} + +int32_t persistGlobalConfig(const char *path, int32_t version) { + // TODO: just tmp ,refactor later + int32_t code = 0; + char *buffer = NULL; + TdFilePtr pFile = NULL; + char filepath[CONFIG_FILE_LEN] = {0}; + char filename[CONFIG_FILE_LEN] = {0}; + snprintf(filepath, sizeof(filepath), "%s%sconfig", path, TD_DIRSEP); + snprintf(filename, sizeof(filename), "%s%sconfig%sglobal.json", path, TD_DIRSEP, TD_DIRSEP); + + // TODO(beryl) need to check if the file is existed + if (taosMkDir(filepath) != 0) { + code = TAOS_SYSTEM_ERROR(errno); + uError("failed to create dir:%s since %s", filepath, tstrerror(code)); + TAOS_RETURN(code); + } + + TdFilePtr pConfigFile = + taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); + + if (pConfigFile == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + uError("failed to open file:%s since %s", filename, tstrerror(code)); + TAOS_RETURN(code); + } + char *serialized = NULL; + code = globalConfigSerialize(version, cfgGetGlobalCfg(tsCfg), &serialized); + if (code != TSDB_CODE_SUCCESS) { + uError("failed to serialize local config since %s", tstrerror(code)); + TAOS_RETURN(code); + } + taosWriteFile(pConfigFile, serialized, strlen(serialized)); + return TSDB_CODE_SUCCESS; +} + +int32_t persistLocalConfig(const char *path) { + // TODO: just tmp ,refactor later + int32_t code = 0; + char *buffer = NULL; + TdFilePtr pFile = NULL; + char filepath[CONFIG_FILE_LEN] = {0}; + char filename[CONFIG_FILE_LEN] = {0}; + snprintf(filepath, sizeof(filepath), "%s%sconfig", path, TD_DIRSEP); + snprintf(filename, sizeof(filename), "%s%sconfig%slocal.json", path, TD_DIRSEP, TD_DIRSEP); + + // TODO(beryl) need to check if the file is existed + if (taosMkDir(filepath) != 0) { + code = TAOS_SYSTEM_ERROR(errno); + uError("failed to create dir:%s since %s", filepath, tstrerror(code)); + TAOS_RETURN(code); + } + + TdFilePtr pConfigFile = + taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); + + if (pConfigFile == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + uError("failed to open file:%s since %s", filename, tstrerror(code)); + TAOS_RETURN(code); + } + char *serialized = NULL; + code = localConfigSerialize(cfgGetLocalCfg(tsCfg), &serialized); + if (code != TSDB_CODE_SUCCESS) { + uError("failed to serialize local config since %s", tstrerror(code)); + TAOS_RETURN(code); + } + taosWriteFile(pConfigFile, serialized, strlen(serialized)); + return TSDB_CODE_SUCCESS; +} diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index cad9154643..27484459aa 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -45,18 +45,7 @@ static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) { return code; } -static void mmBuildConfigForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInput, SMnodeOpt *pOption) { - pOption->deploy = true; - pOption->msgCb = pMgmt->msgCb; - pOption->dnodeId = pMgmt->pData->dnodeId; - pOption->selfIndex = 0; - pOption->numOfReplicas = 1; - pOption->numOfTotalReplicas = 1; - pOption->replicas[0].id = 1; - pOption->replicas[0].port = tsServerPort; - tstrncpy(pOption->replicas[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN); - pOption->lastIndex = SYNC_INDEX_INVALID; -} +static void mmBuildConfigForDeploy(SMnodeMgmt *pMgmt) { persistGlobalConfig(pMgmt->path, 0); } static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInput, SMnodeOpt *pOption) { pOption->deploy = true; @@ -133,7 +122,7 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { dInfo("mnode start to deploy"); pMgmt->pData->dnodeId = 1; mmBuildOptionForDeploy(pMgmt, pInput, &option); - + mmBuildConfigForDeploy(pMgmt); } else { dInfo("mnode start to open"); mmBuildOptionForOpen(pMgmt, &option); From df2331d5e7dcb2c090a677464a6c5ff29b007f67 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 7 Nov 2024 15:29:33 +0800 Subject: [PATCH 09/99] Add config req and config thread. --- include/common/tglobal.h | 4 +- include/common/tmsg.h | 8 ++++ include/common/tmsgdef.h | 1 + source/common/src/tglobal.c | 2 + source/common/src/tmsg.c | 40 ++++++++++++++-- source/dnode/mgmt/mgmt_dnode/inc/dmInt.h | 2 + source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 48 +++++++++++++++++++ source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 52 +++++++++++++++++++++ 8 files changed, 152 insertions(+), 5 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index c8dc12ed3b..56a258b01e 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -49,6 +49,8 @@ extern char tsLocalEp[]; extern char tsVersionName[]; extern uint16_t tsServerPort; extern int32_t tsVersion; +int32_t configVersion; +int32_t configInited; extern int32_t tsStatusInterval; extern int32_t tsNumOfSupportVnodes; extern char tsEncryptAlgorithm[]; @@ -154,7 +156,7 @@ extern bool tsEnableCrashReport; extern char *tsTelemUri; extern char *tsClientCrashReportUri; extern char *tsSvrCrashReportUri; -extern int8_t tsSafetyCheckLevel; +extern int8_t tsSafetyCheckLevel; enum { TSDB_SAFETY_CHECK_LEVELL_NEVER = 0, TSDB_SAFETY_CHECK_LEVELL_NORMAL = 1, diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 53366f6f9d..c5bca8a5f7 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1806,6 +1806,14 @@ int32_t tSerializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq); int32_t tDeserializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq); void tFreeSStatusReq(SStatusReq* pReq); +typedef struct { + int32_t cver; +} SConfigReq; + +int32_t tSerializeSConfigReq(void* buf, int32_t bufLen, SConfigReq* pReq); +int32_t tDeserializeSConfigReq(void* buf, int32_t bufLen, SConfigReq* pReq); +// void tFreeSStatusReq(SStatusReq* pReq); + typedef struct { int32_t dnodeId; char machineId[TSDB_MACHINE_ID_LEN + 1]; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index c22a3da5ad..93bfe306b6 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -260,6 +260,7 @@ TD_DEF_MSG_TYPE(TDMT_MND_STREAM_TASK_RESET, "stream-reset-tasks", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_UPDATE_DNODE_INFO, "update-dnode-info", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_AUDIT, "audit", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CONFIG, "init-config", NULL, NULL) TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG) TD_NEW_MSG_SEG(TDMT_VND_MSG) // 2<<8 diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 7eab064eea..c83e3e8db2 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -43,6 +43,8 @@ char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port char tsVersionName[16] = "community"; uint16_t tsServerPort = 6030; int32_t tsVersion = 30000000; +int32_t configVersion = 0; +int32_t configInited = 0; int32_t tsStatusInterval = 1; // second int32_t tsNumOfSupportVnodes = 256; char tsEncryptAlgorithm[16] = {0}; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 458badc764..ab63d8083c 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1600,6 +1600,38 @@ _exit: return code; } +int32_t tSerializeSConfigReq(void *buf, int32_t bufLen, SConfigReq *pReq) { + SEncoder encoder = {0}; + int32_t code = 0; + int32_t lino; + int32_t tlen; + tEncoderInit(&encoder, buf, bufLen); + TAOS_CHECK_EXIT(tStartEncode(&encoder)); + TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->cver)); + tEndEncode(&encoder); +_exit: + if (code) { + tlen = code; + } else { + tlen = encoder.pos; + } + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSConfigReq(void *buf, int32_t bufLen, SConfigReq *pReq) { + SDecoder decoder = {0}; + int32_t code = 0; + int32_t lino; + tDecoderInit(&decoder, buf, bufLen); + TAOS_CHECK_EXIT(tStartDecode(&decoder)); + TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->cver)); + tEndDecode(&decoder); +_exit: + tDecoderClear(&decoder); + return code; +} + void tFreeSStatusReq(SStatusReq *pReq) { taosArrayDestroy(pReq->pVloads); } int32_t tSerializeSDnodeInfoReq(void *buf, int32_t bufLen, SDnodeInfoReq *pReq) { @@ -2221,10 +2253,10 @@ int32_t tDeserializeRetrieveAnalAlgoRsp(void *buf, int32_t bufLen, SRetrieveAnal int32_t lino; tDecoderInit(&decoder, buf, bufLen); - int32_t numOfAlgos = 0; - int32_t nameLen; - int32_t type; - char name[TSDB_ANAL_ALGO_KEY_LEN]; + int32_t numOfAlgos = 0; + int32_t nameLen; + int32_t type; + char name[TSDB_ANAL_ALGO_KEY_LEN]; SAnalUrl url = {0}; TAOS_CHECK_EXIT(tStartDecode(&decoder)); diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index cbf1959e75..47bbb88ee1 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -28,6 +28,7 @@ typedef struct SDnodeMgmt { const char *path; const char *name; TdThread statusThread; + TdThread configThread; TdThread statusInfoThread; TdThread notifyThread; TdThread monitorThread; @@ -50,6 +51,7 @@ typedef struct SDnodeMgmt { // dmHandle.c SArray *dmGetMsgHandles(); void dmSendStatusReq(SDnodeMgmt *pMgmt); +void dmSendConfigReq(SDnodeMgmt *pMgmt); void dmUpdateStatusInfo(SDnodeMgmt *pMgmt); void dmSendNotifyReq(SDnodeMgmt *pMgmt, SNotifyReq *pReq); int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index d6b792ca74..bda9dc3944 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -283,6 +283,54 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { dmProcessStatusRsp(pMgmt, &rpcRsp); } +void dmSendConfigReq(SDnodeMgmt *pMgmt) { + int32_t code = 0; + SConfigReq req = {0}; + + req.cver = configVersion; + dDebug("send config req to mnode, configVersion:%d", req.cver); + + int32_t contLen = tSerializeSConfigReq(NULL, 0, &req); + if (contLen < 0) { + dError("failed to serialize status req since %s", tstrerror(contLen)); + return; + } + + void *pHead = rpcMallocCont(contLen); + contLen = tSerializeSConfigReq(pHead, contLen, &req); + if (contLen < 0) { + rpcFreeCont(pHead); + dError("failed to serialize status req since %s", tstrerror(contLen)); + return; + } + + SRpcMsg rpcMsg = {.pCont = pHead, + .contLen = contLen, + .msgType = TDMT_MND_CONFIG, + .info.ahandle = 0, + .info.notFreeAhandle = 1, + .info.refId = 0, + .info.noResp = 0, + .info.handle = 0}; + SRpcMsg rpcRsp = {0}; + + SEpSet epSet = {0}; + int8_t epUpdated = 0; + (void)dmGetMnodeEpSet(pMgmt->pData, &epSet); + + dDebug("send status req to mnode, statusSeq:%d, begin to send rpc msg", pMgmt->statusSeq); + code = + rpcSendRecvWithTimeout(pMgmt->msgCb.statusRpc, &epSet, &rpcMsg, &rpcRsp, &epUpdated, tsStatusInterval * 5 * 1000); + if (code != 0) { + dError("failed to send status req since %s", tstrerror(code)); + return; + } + + if (rpcRsp.code != 0) { + } else { + } +} + void dmUpdateStatusInfo(SDnodeMgmt *pMgmt) { SMonVloadInfo vinfo = {0}; dDebug("begin to get vnode loads"); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 7f802f3837..9c6cd4fdd6 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -47,6 +47,35 @@ static void *dmStatusThreadFp(void *param) { return NULL; } +static void *dmConfigThreadFp(void *param) { + SDnodeMgmt *pMgmt = param; + int64_t lastTime = taosGetTimestampMs(); + setThreadName("dnode-config"); + + int32_t upTimeCount = 0; + int64_t upTime = 0; + + while (1) { + taosMsleep(200); + if (pMgmt->pData->dropped || pMgmt->pData->stopped || configInited) break; + + int64_t curTime = taosGetTimestampMs(); + if (curTime < lastTime) lastTime = curTime; + float interval = (curTime - lastTime) / 1000.0f; + if (interval >= tsStatusInterval) { + dmSendConfigReq(pMgmt); + lastTime = curTime; + + if ((upTimeCount = ((upTimeCount + 1) & 63)) == 0) { + upTime = taosGetOsUptime() - tsDndStartOsUptime; + tsDndUpTime = TMAX(tsDndUpTime, upTime); + } + } + } + + return NULL; +} + static void *dmStatusInfoThreadFp(void *param) { SDnodeMgmt *pMgmt = param; int64_t lastTime = taosGetTimestampMs(); @@ -309,6 +338,22 @@ int32_t dmStartStatusThread(SDnodeMgmt *pMgmt) { return 0; } +int32_t dmStartConfigThread(SDnodeMgmt *pMgmt) { + int32_t code = 0; + TdThreadAttr thAttr; + (void)taosThreadAttrInit(&thAttr); + (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + if (taosThreadCreate(&pMgmt->configThread, &thAttr, dmConfigThreadFp, pMgmt) != 0) { + code = TAOS_SYSTEM_ERROR(errno); + dError("failed to create config thread since %s", tstrerror(code)); + return code; + } + + (void)taosThreadAttrDestroy(&thAttr); + tmsgReportStartup("config-status", "initialized"); + return 0; +} + int32_t dmStartStatusInfoThread(SDnodeMgmt *pMgmt) { int32_t code = 0; TdThreadAttr thAttr; @@ -332,6 +377,13 @@ void dmStopStatusThread(SDnodeMgmt *pMgmt) { } } +void dmStopConfigThread(SDnodeMgmt *pMgmt) { + if (taosCheckPthreadValid(pMgmt->configThread)) { + (void)taosThreadJoin(pMgmt->configThread, NULL); + taosThreadClear(&pMgmt->configThread); + } +} + void dmStopStatusInfoThread(SDnodeMgmt *pMgmt) { if (taosCheckPthreadValid(pMgmt->statusInfoThread)) { (void)taosThreadJoin(pMgmt->statusInfoThread, NULL); From 8e2816038e2ac852fd6938af6b89e927ab1c9a66 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 7 Nov 2024 16:19:04 +0800 Subject: [PATCH 10/99] Resolving compilation errors. --- include/common/tglobal.h | 4 ++-- source/common/src/tglobal.c | 5 ++--- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 56a258b01e..0bd15c3418 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -49,8 +49,8 @@ extern char tsLocalEp[]; extern char tsVersionName[]; extern uint16_t tsServerPort; extern int32_t tsVersion; -int32_t configVersion; -int32_t configInited; +extern int32_t tsConfigVersion; +extern int32_t tsConfigInited; extern int32_t tsStatusInterval; extern int32_t tsNumOfSupportVnodes; extern char tsEncryptAlgorithm[]; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c83e3e8db2..2919c2cb9f 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -43,8 +43,8 @@ char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port char tsVersionName[16] = "community"; uint16_t tsServerPort = 6030; int32_t tsVersion = 30000000; -int32_t configVersion = 0; -int32_t configInited = 0; +int32_t tsConfigVersion = 0; +int32_t tsConfigInited = 0; int32_t tsStatusInterval = 1; // second int32_t tsNumOfSupportVnodes = 256; char tsEncryptAlgorithm[16] = {0}; @@ -649,7 +649,6 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { TAOS_CHECK_RETURN( cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, -<<<<<<< HEAD CFG_SCOPE_CLIENT, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "metaCacheMaxSize", tsMetaCacheMaxSize, -1, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index bda9dc3944..f21f6a820f 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -287,7 +287,7 @@ void dmSendConfigReq(SDnodeMgmt *pMgmt) { int32_t code = 0; SConfigReq req = {0}; - req.cver = configVersion; + req.cver = tsConfigVersion; dDebug("send config req to mnode, configVersion:%d", req.cver); int32_t contLen = tSerializeSConfigReq(NULL, 0, &req); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 9c6cd4fdd6..d160f6584c 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -57,7 +57,7 @@ static void *dmConfigThreadFp(void *param) { while (1) { taosMsleep(200); - if (pMgmt->pData->dropped || pMgmt->pData->stopped || configInited) break; + if (pMgmt->pData->dropped || pMgmt->pData->stopped || tsConfigInited) break; int64_t curTime = taosGetTimestampMs(); if (curTime < lastTime) lastTime = curTime; From 8355be28d9caf9d19258c6dcf07ebd6ed5f63b20 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Fri, 8 Nov 2024 16:11:04 +0800 Subject: [PATCH 11/99] add encode and decode for config --- include/common/tglobal.h | 6 +- include/common/tmsg.h | 15 +++- include/util/tencode.h | 13 ++++ source/common/src/tglobal.c | 78 +++++++++++++++++++ source/common/src/tmsg.c | 6 ++ source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 + source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 1 + source/dnode/mnode/impl/src/mndDnode.c | 83 +++++++++++++++------ 8 files changed, 178 insertions(+), 26 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 0bd15c3418..ce81356f86 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -19,6 +19,7 @@ #include "tarray.h" #include "tconfig.h" #include "tdef.h" +#include "tmsg.h" #ifdef __cplusplus extern "C" { @@ -49,6 +50,7 @@ extern char tsLocalEp[]; extern char tsVersionName[]; extern uint16_t tsServerPort; extern int32_t tsVersion; +extern int32_t tsForceReadConfig; extern int32_t tsConfigVersion; extern int32_t tsConfigInited; extern int32_t tsStatusInterval; @@ -294,8 +296,8 @@ int32_t taosSetSlowLogScope(char *pScopeStr, int32_t *pScope); int32_t persistGlobalConfig(const char *path, int32_t version); int32_t persistLocalConfig(const char *path); int32_t localConfigSerialize(SArray *array, char **serialized); -int32_t tSerializeSConfigArray(void *buf, int32_t bufLen, SArray *array); -int32_t tDeserializeSConfigArray(void *buf, int32_t bufLen, SArray *pReq); +int32_t tSerializeSConfigArray(SEncoder *pEncoder, int32_t bufLen, SArray *array); +int32_t tDeserializeSConfigArray(SDecoder *pDecoder, int32_t bufLen, SArray *array); #ifdef __cplusplus } #endif diff --git a/include/common/tmsg.h b/include/common/tmsg.h index c5bca8a5f7..6cbf814cb9 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1807,12 +1807,14 @@ int32_t tDeserializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq); void tFreeSStatusReq(SStatusReq* pReq); typedef struct { + int32_t forceReadConfig; int32_t cver; + SArray* array; } SConfigReq; int32_t tSerializeSConfigReq(void* buf, int32_t bufLen, SConfigReq* pReq); int32_t tDeserializeSConfigReq(void* buf, int32_t bufLen, SConfigReq* pReq); -// void tFreeSStatusReq(SStatusReq* pReq); +void tFreeSStatusReq(SStatusReq* pReq); typedef struct { int32_t dnodeId; @@ -1891,6 +1893,17 @@ int32_t tSerializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp); int32_t tDeserializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp); void tFreeSStatusRsp(SStatusRsp* pRsp); +typedef struct { + int32_t forceReadConfig; + int32_t isConifgVerified; + int32_t isVersionVerified; + SArray* pArray; +} SConfigRsp; + +int32_t tSerializeSConfigRsp(void* buf, int32_t bufLen, SConfigRsp* pRsp); +int32_t tDeserializeSConfigRsp(void* buf, int32_t bufLen, SConfigRsp* pRsp); +void tFreeSConfigRsp(SConfigRsp* pRsp); + typedef struct { int32_t reserved; } SMTimerReq; diff --git a/include/util/tencode.h b/include/util/tencode.h index b66e79fa60..89a675d817 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -88,6 +88,7 @@ static int32_t tEncodeU64v(SEncoder* pCoder, uint64_t val); static int32_t tEncodeI64v(SEncoder* pCoder, int64_t val); static int32_t tEncodeFloat(SEncoder* pCoder, float val); static int32_t tEncodeDouble(SEncoder* pCoder, double val); +static int32_t tEncodeBool(SEncoder* pCoder, bool val); static int32_t tEncodeBinary(SEncoder* pCoder, const uint8_t* val, uint32_t len); static int32_t tEncodeBinaryEx(SEncoder* pCoder, const uint8_t* val, uint32_t len); static int32_t tEncodeCStrWithLen(SEncoder* pCoder, const char* val, uint32_t len); @@ -116,6 +117,7 @@ static int32_t tDecodeU64v(SDecoder* pCoder, uint64_t* val); static int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val); static int32_t tDecodeFloat(SDecoder* pCoder, float* val); static int32_t tDecodeDouble(SDecoder* pCoder, double* val); +static int32_t tDecodeBool(SDecoder* pCoder, bool* val); static int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len); static int32_t tDecodeCStrAndLen(SDecoder* pCoder, char** val, uint32_t* len); static int32_t tDecodeCStr(SDecoder* pCoder, char** val); @@ -205,6 +207,8 @@ static FORCE_INLINE int32_t tEncodeDouble(SEncoder* pCoder, double val) { return tEncodeU64(pCoder, v.ui); } +static int32_t tEncodeBool(SEncoder* pCoder, bool val) { return tEncodeU8(pCoder, val ? 1 : 0); } + static FORCE_INLINE int32_t tEncodeBinary(SEncoder* pCoder, const uint8_t* val, uint32_t len) { TAOS_CHECK_RETURN(tEncodeU32v(pCoder, len)); if (len) { @@ -391,6 +395,15 @@ static FORCE_INLINE int32_t tDecodeDouble(SDecoder* pCoder, double* val) { return 0; } +static int32_t tDecodeBool(SDecoder* pCoder, bool* val) { + uint8_t v; + TAOS_CHECK_RETURN(tDecodeU8(pCoder, &v)); + if (val) { + *val = v ? true : false; + } + return 0; +} + static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len) { uint32_t length = 0; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 2919c2cb9f..0fac0c9c8e 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -23,6 +23,7 @@ #include "tlog.h" #include "tmisce.h" #include "tunit.h" +#include "tutil.h" #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #include "cus_name.h" @@ -43,6 +44,7 @@ char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port char tsVersionName[16] = "community"; uint16_t tsServerPort = 6030; int32_t tsVersion = 30000000; +int32_t tsForceReadConfig = 0; int32_t tsConfigVersion = 0; int32_t tsConfigInited = 0; int32_t tsStatusInterval = 1; // second @@ -2676,3 +2678,79 @@ int32_t persistLocalConfig(const char *path) { taosWriteFile(pConfigFile, serialized, strlen(serialized)); return TSDB_CODE_SUCCESS; } + +int32_t tSerializeSConfigArray(SEncoder *pEncoder, int32_t bufLen, SArray *array) { + int32_t code = 0; + int32_t lino = 0; + int sz = taosArrayGetSize(array); + + for (int i = 0; i < sz; i++) { + SConfigItem *item = (SConfigItem *)taosArrayGet(array, i); + switch (item->dtype) { + { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + TAOS_CHECK_EXIT(tEncodeBool(pEncoder, item->bval)); + break; + case CFG_DTYPE_INT32: + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, item->i32)); + break; + case CFG_DTYPE_INT64: + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, item->i64)); + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + TAOS_CHECK_EXIT(tEncodeFloat(pEncoder, item->fval)); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, item->str)); + break; + } + } + } +_exit: + return code; +} + +int32_t tDeserializeSConfigArray(SDecoder *pDecoder, int32_t bufLen, SArray *array) { + int32_t code = 0; + int32_t lino = 0; + int sz = taosArrayGetSize(array); + + for (int i = 0; i < sz; i++) { + SConfigItem *item = (SConfigItem *)taosArrayGet(array, i); + switch (item->dtype) { + { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + TAOS_CHECK_EXIT(tDecodeBool(pDecoder, &item->bval)); + break; + case CFG_DTYPE_INT32: + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &item->i32)); + break; + case CFG_DTYPE_INT64: + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &item->i64)); + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + TAOS_CHECK_EXIT(tDecodeFloat(pDecoder, &item->fval)); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + TAOS_CHECK_EXIT(tDecodeCStr(pDecoder, &item->str)); + break; + } + } + } +_exit: + return code; +} diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index ab63d8083c..40750a1b86 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1600,6 +1600,7 @@ _exit: return code; } + int32_t tSerializeSConfigReq(void *buf, int32_t bufLen, SConfigReq *pReq) { SEncoder encoder = {0}; int32_t code = 0; @@ -1607,6 +1608,11 @@ int32_t tSerializeSConfigReq(void *buf, int32_t bufLen, SConfigReq *pReq) { int32_t tlen; tEncoderInit(&encoder, buf, bufLen); TAOS_CHECK_EXIT(tStartEncode(&encoder)); + TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->forceReadConfig)); + if (pReq->forceReadConfig) { + } else { + TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->cver)); + } TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->cver)); tEndEncode(&encoder); _exit: diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index f21f6a820f..a2aa16e2af 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -288,6 +288,8 @@ void dmSendConfigReq(SDnodeMgmt *pMgmt) { SConfigReq req = {0}; req.cver = tsConfigVersion; + req.forceReadConfig = tsForceReadConfig; + req.array = cfgGetGlobalCfg(tsCfg); dDebug("send config req to mnode, configVersion:%d", req.cver); int32_t contLen = tSerializeSConfigReq(NULL, 0, &req); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 0d804eadf0..7b01609c29 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -200,6 +200,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_CONN, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_NOTIFY, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 24ae8382f9..8e042e2091 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -14,11 +14,11 @@ */ #define _DEFAULT_SOURCE -#include "mndDnode.h" #include #include "audit.h" #include "mndCluster.h" #include "mndDb.h" +#include "mndDnode.h" #include "mndMnode.h" #include "mndPrivilege.h" #include "mndQnode.h" @@ -83,6 +83,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq); static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq); static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessStatusReq(SRpcMsg *pReq); +static int32_t mndProcessConfigReq(SRpcMsg *pReq); static int32_t mndProcessNotifyReq(SRpcMsg *pReq); static int32_t mndProcessRestoreDnodeReq(SRpcMsg *pReq); static int32_t mndProcessStatisReq(SRpcMsg *pReq); @@ -121,6 +122,7 @@ int32_t mndInitDnode(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_CONFIG_DNODE, mndProcessConfigDnodeReq); mndSetMsgHandle(pMnode, TDMT_DND_CONFIG_DNODE_RSP, mndProcessConfigDnodeRsp); mndSetMsgHandle(pMnode, TDMT_MND_STATUS, mndProcessStatusReq); + mndSetMsgHandle(pMnode, TDMT_MND_CONFIG, mndProcessConfigReq); mndSetMsgHandle(pMnode, TDMT_MND_NOTIFY, mndProcessNotifyReq); mndSetMsgHandle(pMnode, TDMT_MND_DNODE_LIST, mndProcessDnodeListReq); mndSetMsgHandle(pMnode, TDMT_MND_SHOW_VARIABLES, mndProcessShowVariablesReq); @@ -462,7 +464,7 @@ int32_t mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) { dInfo.isMnode = 0; } - if(taosArrayPush(pDnodeInfo, &dInfo) == NULL){ + if (taosArrayPush(pDnodeInfo, &dInfo) == NULL) { code = terrno; sdbCancelFetch(pSdb, pIter); break; @@ -471,12 +473,12 @@ int32_t mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) { TAOS_RETURN(code); } -#define CHECK_MONITOR_PARA(para,err) \ -if (pCfg->monitorParas.para != para) { \ - mError("dnode:%d, para:%d inconsistent with cluster:%d", pDnode->id, pCfg->monitorParas.para, para); \ - terrno = err; \ - return err;\ -} +#define CHECK_MONITOR_PARA(para, err) \ + if (pCfg->monitorParas.para != para) { \ + mError("dnode:%d, para:%d inconsistent with cluster:%d", pDnode->id, pCfg->monitorParas.para, para); \ + terrno = err; \ + return err; \ + } static int32_t mndCheckClusterCfgPara(SMnode *pMnode, SDnodeObj *pDnode, const SClusterCfg *pCfg) { CHECK_MONITOR_PARA(tsEnableMonitor, DND_REASON_STATUS_MONITOR_SWITCH_NOT_MATCH); @@ -487,7 +489,8 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, SDnodeObj *pDnode, const S CHECK_MONITOR_PARA(tsSlowLogScope, DND_REASON_STATUS_MONITOR_SLOW_LOG_SCOPE_NOT_MATCH); if (0 != strcasecmp(pCfg->monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb)) { - mError("dnode:%d, tsSlowLogExceptDb:%s inconsistent with cluster:%s", pDnode->id, pCfg->monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb); + mError("dnode:%d, tsSlowLogExceptDb:%s inconsistent with cluster:%s", pDnode->id, + pCfg->monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb); terrno = TSDB_CODE_DNODE_INVALID_MONITOR_PARAS; return DND_REASON_STATUS_MONITOR_NOT_MATCH; } @@ -583,8 +586,8 @@ static bool mndUpdateMnodeState(SMnodeObj *pObj, SMnodeLoad *pMload) { return stateChanged; } -extern char* tsMonFwUri; -extern char* tsMonSlowLogUri; +extern char *tsMonFwUri; +extern char *tsMonSlowLogUri; static int32_t mndProcessStatisReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStatisReq statisReq = {0}; @@ -596,9 +599,9 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { mInfo("process statis req,\n %s", statisReq.pCont); } - if (statisReq.type == MONITOR_TYPE_COUNTER){ + if (statisReq.type == MONITOR_TYPE_COUNTER) { monSendContent(statisReq.pCont, tsMonFwUri); - }else if(statisReq.type == MONITOR_TYPE_SLOW_LOG){ + } else if (statisReq.type == MONITOR_TYPE_SLOW_LOG) { monSendContent(statisReq.pCont, tsMonSlowLogUri); } @@ -920,6 +923,40 @@ _OVER: return mndUpdClusterInfo(pReq); } +static int32_t mndProcessConfigReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SConfigReq configReq = {0}; + SDnodeObj *pDnode = NULL; + int32_t code = -1; + + tDeserializeSConfigReq(pReq->pCont, pReq->contLen, &configReq); + + SStatusRsp statusRsp = {0}; + statusRsp.statusSeq++; + statusRsp.dnodeCfg.dnodeId = pDnode->id; + statusRsp.dnodeCfg.clusterId = pMnode->clusterId; + statusRsp.pDnodeEps = taosArrayInit(mndGetDnodeSize(pMnode), sizeof(SDnodeEp)); + if (statusRsp.pDnodeEps == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + mndGetDnodeEps(pMnode, statusRsp.pDnodeEps); + statusRsp.ipWhiteVer = pMnode->ipWhiteVer; + + int32_t contLen = tSerializeSStatusRsp(NULL, 0, &statusRsp); + void *pHead = rpcMallocCont(contLen); + contLen = tSerializeSStatusRsp(pHead, contLen, &statusRsp); + taosArrayDestroy(statusRsp.pDnodeEps); + if (contLen < 0) { + code = contLen; + goto _OVER; + } +_OVER: + mndReleaseDnode(pMnode, pDnode); + return mndUpdClusterInfo(pReq); +} + static int32_t mndProcessNotifyReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SNotifyReq notifyReq = {0}; @@ -1058,27 +1095,27 @@ _OVER: TAOS_RETURN(code); } -static void getSlowLogScopeString(int32_t scope, char* result){ - if(scope == SLOW_LOG_TYPE_NULL) { +static void getSlowLogScopeString(int32_t scope, char *result) { + if (scope == SLOW_LOG_TYPE_NULL) { (void)strcat(result, "NONE"); return; } - while(scope > 0){ - if(scope & SLOW_LOG_TYPE_QUERY) { + while (scope > 0) { + if (scope & SLOW_LOG_TYPE_QUERY) { (void)strcat(result, "QUERY"); scope &= ~SLOW_LOG_TYPE_QUERY; - } else if(scope & SLOW_LOG_TYPE_INSERT) { + } else if (scope & SLOW_LOG_TYPE_INSERT) { (void)strcat(result, "INSERT"); scope &= ~SLOW_LOG_TYPE_INSERT; - } else if(scope & SLOW_LOG_TYPE_OTHERS) { + } else if (scope & SLOW_LOG_TYPE_OTHERS) { (void)strcat(result, "OTHERS"); scope &= ~SLOW_LOG_TYPE_OTHERS; - } else{ + } else { (void)printf("invalid slow log scope:%d", scope); return; } - if(scope > 0) { + if (scope > 0) { (void)strcat(result, "|"); } } @@ -1439,7 +1476,7 @@ _OVER: static int32_t mndMCfg2DCfg(SMCfgDnodeReq *pMCfgReq, SDCfgDnodeReq *pDCfgReq) { int32_t code = 0; - char *p = pMCfgReq->config; + char *p = pMCfgReq->config; while (*p) { if (*p == ' ') { break; @@ -1537,7 +1574,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); #endif } else { - TAOS_CHECK_GOTO (mndMCfg2DCfg(&cfgReq, &dcfgReq), NULL, _err_out); + TAOS_CHECK_GOTO(mndMCfg2DCfg(&cfgReq, &dcfgReq), NULL, _err_out); if (strlen(dcfgReq.config) > TSDB_DNODE_CONFIG_LEN) { mError("dnode:%d, failed to config since config is too long", cfgReq.dnodeId); code = TSDB_CODE_INVALID_CFG; From f931119e14bcbd4d333a98db7ef6e231e68dfd91 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 11 Nov 2024 11:33:10 +0800 Subject: [PATCH 12/99] Add encode and decode rpc config msg --- include/common/tglobal.h | 8 +++- include/common/tmsg.h | 2 +- source/common/src/tglobal.c | 54 +++++++++++++++++++++++++- source/common/src/tmsg.c | 54 ++++++++++++++++++++++++-- source/dnode/mnode/impl/src/mndDnode.c | 37 +++++++++++------- 5 files changed, 131 insertions(+), 24 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index ce81356f86..6353443ccc 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -42,6 +42,8 @@ typedef enum { DND_CS_MNODE_WAL = 8, } EEncryptScope; +extern SConfig *tsCfg; + // cluster extern char tsFirst[]; extern char tsSecond[]; @@ -296,8 +298,10 @@ int32_t taosSetSlowLogScope(char *pScopeStr, int32_t *pScope); int32_t persistGlobalConfig(const char *path, int32_t version); int32_t persistLocalConfig(const char *path); int32_t localConfigSerialize(SArray *array, char **serialized); -int32_t tSerializeSConfigArray(SEncoder *pEncoder, int32_t bufLen, SArray *array); -int32_t tDeserializeSConfigArray(SDecoder *pDecoder, int32_t bufLen, SArray *array); +int32_t tSerializeSConfigArray(SEncoder *pEncoder, SArray *array); +int32_t tDeserializeSConfigArray(SDecoder *pDecoder, SArray *array); + +int32_t compareSConfigItemArrays(SArray *mArray, const SArray *dArray, SArray *diffArray); #ifdef __cplusplus } #endif diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 6cbf814cb9..ec257b6b6f 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1897,7 +1897,7 @@ typedef struct { int32_t forceReadConfig; int32_t isConifgVerified; int32_t isVersionVerified; - SArray* pArray; + SArray* array; } SConfigRsp; int32_t tSerializeSConfigRsp(void* buf, int32_t bufLen, SConfigRsp* pRsp); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 0fac0c9c8e..fc3a9652ef 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -2679,13 +2679,14 @@ int32_t persistLocalConfig(const char *path) { return TSDB_CODE_SUCCESS; } -int32_t tSerializeSConfigArray(SEncoder *pEncoder, int32_t bufLen, SArray *array) { +int32_t tSerializeSConfigArray(SEncoder *pEncoder, SArray *array) { int32_t code = 0; int32_t lino = 0; int sz = taosArrayGetSize(array); for (int i = 0; i < sz; i++) { SConfigItem *item = (SConfigItem *)taosArrayGet(array, i); + TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, item->name)); switch (item->dtype) { { case CFG_DTYPE_NONE: @@ -2717,13 +2718,14 @@ _exit: return code; } -int32_t tDeserializeSConfigArray(SDecoder *pDecoder, int32_t bufLen, SArray *array) { +int32_t tDeserializeSConfigArray(SDecoder *pDecoder, SArray *array) { int32_t code = 0; int32_t lino = 0; int sz = taosArrayGetSize(array); for (int i = 0; i < sz; i++) { SConfigItem *item = (SConfigItem *)taosArrayGet(array, i); + TAOS_CHECK_EXIT(tDecodeCStr(pDecoder, &item->name)); switch (item->dtype) { { case CFG_DTYPE_NONE: @@ -2754,3 +2756,51 @@ int32_t tDeserializeSConfigArray(SDecoder *pDecoder, int32_t bufLen, SArray *arr _exit: return code; } + +bool compareSConfigItem(const SConfigItem *item1, const SConfigItem *item2) { + switch (item1->dtype) { + case CFG_DTYPE_BOOL: + if (item1->bval != item2->bval) return false; + break; + case CFG_DTYPE_FLOAT: + if (item1->fval != item2->fval) return false; + break; + case CFG_DTYPE_INT32: + if (item1->i32 != item2->i32) return false; + break; + case CFG_DTYPE_INT64: + if (item1->i64 != item2->i64) return false; + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + if (strcmp(item1->str, item2->str) != 0) return false; + break; + default: + return false; + } + return true; +} + +int32_t compareSConfigItemArrays(SArray *mArray, const SArray *dArray, SArray *diffArray) { + int32_t code = 0; + int32_t msz = taosArrayGetSize(mArray); + int32_t dsz = taosArrayGetSize(dArray); + + if (msz != dsz) { + return TSDB_CODE_FAILED; + } + + for (int i = 0; i < msz; i++) { + SConfigItem *mItem = (SConfigItem *)taosArrayGet(mArray, i); + SConfigItem *dItem = (SConfigItem *)taosArrayGet(dArray, i); + if (!compareSConfigItem(mItem, dItem)) { + code = TSDB_CODE_FAILED; + taosArrayPush(diffArray, mItem); + } + } + + return code; +} \ No newline at end of file diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 40750a1b86..9e94c92fac 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE +#include "tglobal.h" #include "tmsg.h" #undef TD_MSG_NUMBER_ @@ -1600,6 +1601,7 @@ _exit: return code; } +void tFreeSStatusReq(SStatusReq *pReq) { taosArrayDestroy(pReq->pVloads); } int32_t tSerializeSConfigReq(void *buf, int32_t bufLen, SConfigReq *pReq) { SEncoder encoder = {0}; @@ -1608,12 +1610,11 @@ int32_t tSerializeSConfigReq(void *buf, int32_t bufLen, SConfigReq *pReq) { int32_t tlen; tEncoderInit(&encoder, buf, bufLen); TAOS_CHECK_EXIT(tStartEncode(&encoder)); + TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->cver)); TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->forceReadConfig)); if (pReq->forceReadConfig) { - } else { - TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->cver)); + TAOS_CHECK_EXIT(tSerializeSConfigArray(&encoder, pReq->array)); } - TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->cver)); tEndEncode(&encoder); _exit: if (code) { @@ -1632,13 +1633,58 @@ int32_t tDeserializeSConfigReq(void *buf, int32_t bufLen, SConfigReq *pReq) { tDecoderInit(&decoder, buf, bufLen); TAOS_CHECK_EXIT(tStartDecode(&decoder)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->cver)); + TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->forceReadConfig)); + if (pReq->forceReadConfig) { + TAOS_CHECK_EXIT(tDeserializeSConfigArray(&decoder, pReq->array)); + } tEndDecode(&decoder); _exit: tDecoderClear(&decoder); return code; } -void tFreeSStatusReq(SStatusReq *pReq) { taosArrayDestroy(pReq->pVloads); } +int32_t tSerializeSConfigRsp(void *buf, int32_t bufLen, SConfigRsp *pRsp) { + SEncoder encoder = {0}; + int32_t code = 0; + int32_t lino; + int32_t tlen; + tEncoderInit(&encoder, buf, bufLen); + TAOS_CHECK_EXIT(tStartEncode(&encoder)); + TAOS_CHECK_EXIT(tEncodeI32(&encoder, pRsp->forceReadConfig)); + TAOS_CHECK_EXIT(tEncodeI32(&encoder, pRsp->isConifgVerified)); + TAOS_CHECK_EXIT(tEncodeI32(&encoder, pRsp->isVersionVerified)); + if ((!pRsp->isConifgVerified) || (!pRsp->isVersionVerified)) { + TAOS_CHECK_EXIT(tSerializeSConfigArray(&encoder, pRsp->array)); + } + tEndEncode(&encoder); +_exit: + if (code) { + tlen = code; + } else { + tlen = encoder.pos; + } + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSConfigRsp(void *buf, int32_t bufLen, SConfigRsp *pRsp) { + SDecoder decoder = {0}; + int32_t code = 0; + int32_t lino; + tDecoderInit(&decoder, buf, bufLen); + TAOS_CHECK_EXIT(tStartDecode(&decoder)); + TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pRsp->forceReadConfig)); + TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pRsp->isConifgVerified)); + TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pRsp->isVersionVerified)); + if ((!pRsp->isConifgVerified) || (!pRsp->isVersionVerified)) { + TAOS_CHECK_EXIT(tDeserializeSConfigArray(&decoder, pRsp->array)); + } +_exit: + tEndDecode(&decoder); + tDecoderClear(&decoder); + return code; +} +void tFreeSConfigRsp(SConfigRsp *pRsp); int32_t tSerializeSDnodeInfoReq(void *buf, int32_t bufLen, SDnodeInfoReq *pReq) { int32_t code = 0, lino = 0; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 8e042e2091..0cc02ec93c 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -28,6 +28,7 @@ #include "mndUser.h" #include "mndVgroup.h" #include "taos_monitor.h" +#include "tconfig.h" #include "tjson.h" #include "tmisce.h" #include "tunit.h" @@ -930,24 +931,30 @@ static int32_t mndProcessConfigReq(SRpcMsg *pReq) { int32_t code = -1; tDeserializeSConfigReq(pReq->pCont, pReq->contLen, &configReq); - - SStatusRsp statusRsp = {0}; - statusRsp.statusSeq++; - statusRsp.dnodeCfg.dnodeId = pDnode->id; - statusRsp.dnodeCfg.clusterId = pMnode->clusterId; - statusRsp.pDnodeEps = taosArrayInit(mndGetDnodeSize(pMnode), sizeof(SDnodeEp)); - if (statusRsp.pDnodeEps == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _OVER; + SArray *diffArray = taosArrayInit(16, sizeof(SConfigItem)); + SConfigRsp configRsp = {0}; + configRsp.forceReadConfig = configReq.forceReadConfig; + if (configRsp.forceReadConfig) { + // compare config array from configReq with current config array + if (compareSConfigItemArrays(cfgGetGlobalCfg(tsCfg), configReq.array, diffArray)) { + configRsp.array = diffArray; + } else { + configRsp.isConifgVerified = 1; + taosArrayDestroy(diffArray); + } + } else { + configRsp.array = cfgGetGlobalCfg(tsCfg); + if (configReq.cver == tsConfigVersion) { + configRsp.isConifgVerified = 1; + } else { + configRsp.array = cfgGetGlobalCfg(tsCfg); + } } - mndGetDnodeEps(pMnode, statusRsp.pDnodeEps); - statusRsp.ipWhiteVer = pMnode->ipWhiteVer; - - int32_t contLen = tSerializeSStatusRsp(NULL, 0, &statusRsp); + int32_t contLen = tSerializeSConfigRsp(NULL, 0, &configRsp); void *pHead = rpcMallocCont(contLen); - contLen = tSerializeSStatusRsp(pHead, contLen, &statusRsp); - taosArrayDestroy(statusRsp.pDnodeEps); + contLen = tSerializeSConfigRsp(pHead, contLen, &configRsp); + taosArrayDestroy(diffArray); if (contLen < 0) { code = contLen; goto _OVER; From 8a90ebaa389eaa4b170c45a3b9c846d08c454da4 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 11 Nov 2024 16:52:47 +0800 Subject: [PATCH 13/99] add process config rsp. --- include/common/tglobal.h | 4 +- include/common/tmsg.h | 1 + source/common/src/tglobal.c | 109 +++++++++++++++++--- source/common/src/tmsg.c | 4 +- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 44 +++++++- source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 2 + 7 files changed, 145 insertions(+), 21 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 6353443ccc..89aee31b6a 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -295,12 +295,14 @@ void taosLocalCfgForbiddenToChange(char *name, bool *forbidden); int8_t taosGranted(int8_t type); int32_t taosSetSlowLogScope(char *pScopeStr, int32_t *pScope); -int32_t persistGlobalConfig(const char *path, int32_t version); +int32_t persistGlobalConfig(SArray *array, const char *path, int32_t version); int32_t persistLocalConfig(const char *path); int32_t localConfigSerialize(SArray *array, char **serialized); int32_t tSerializeSConfigArray(SEncoder *pEncoder, SArray *array); int32_t tDeserializeSConfigArray(SDecoder *pDecoder, SArray *array); +void printConfigNotMatch(SArray *array); + int32_t compareSConfigItemArrays(SArray *mArray, const SArray *dArray, SArray *diffArray); #ifdef __cplusplus } diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ec257b6b6f..abe6465627 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1897,6 +1897,7 @@ typedef struct { int32_t forceReadConfig; int32_t isConifgVerified; int32_t isVersionVerified; + int32_t cver; SArray* array; } SConfigRsp; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index fc3a9652ef..2da6b9eb2f 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -2609,7 +2609,7 @@ int32_t localConfigSerialize(SArray *array, char **serialized) { return TSDB_CODE_SUCCESS; } -int32_t persistGlobalConfig(const char *path, int32_t version) { +int32_t persistGlobalConfig(SArray *array, const char *path, int32_t version) { // TODO: just tmp ,refactor later int32_t code = 0; char *buffer = NULL; @@ -2635,7 +2635,7 @@ int32_t persistGlobalConfig(const char *path, int32_t version) { TAOS_RETURN(code); } char *serialized = NULL; - code = globalConfigSerialize(version, cfgGetGlobalCfg(tsCfg), &serialized); + code = globalConfigSerialize(version, array, &serialized); if (code != TSDB_CODE_SUCCESS) { uError("failed to serialize local config since %s", tstrerror(code)); TAOS_RETURN(code); @@ -2682,11 +2682,12 @@ int32_t persistLocalConfig(const char *path) { int32_t tSerializeSConfigArray(SEncoder *pEncoder, SArray *array) { int32_t code = 0; int32_t lino = 0; - int sz = taosArrayGetSize(array); - + int32_t sz = taosArrayGetSize(array); + tEncodeI32(pEncoder, sz); for (int i = 0; i < sz; i++) { SConfigItem *item = (SConfigItem *)taosArrayGet(array, i); TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, item->name)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, item->dtype)); switch (item->dtype) { { case CFG_DTYPE_NONE: @@ -2719,36 +2720,38 @@ _exit: } int32_t tDeserializeSConfigArray(SDecoder *pDecoder, SArray *array) { - int32_t code = 0; - int32_t lino = 0; - int sz = taosArrayGetSize(array); - + int32_t code = 0; + int32_t lino = 0; + int32_t sz = 0; + ECfgDataType dtype = CFG_DTYPE_NONE; + tDecodeI32(pDecoder, &sz); for (int i = 0; i < sz; i++) { - SConfigItem *item = (SConfigItem *)taosArrayGet(array, i); - TAOS_CHECK_EXIT(tDecodeCStr(pDecoder, &item->name)); - switch (item->dtype) { + SConfigItem item = {0}; + TAOS_CHECK_EXIT(tDecodeCStr(pDecoder, &item.name)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, (int32_t *)&dtype)); + switch (dtype) { { case CFG_DTYPE_NONE: break; case CFG_DTYPE_BOOL: - TAOS_CHECK_EXIT(tDecodeBool(pDecoder, &item->bval)); + TAOS_CHECK_EXIT(tDecodeBool(pDecoder, &item.bval)); break; case CFG_DTYPE_INT32: - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &item->i32)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &item.i32)); break; case CFG_DTYPE_INT64: - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &item->i64)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &item.i64)); break; case CFG_DTYPE_FLOAT: case CFG_DTYPE_DOUBLE: - TAOS_CHECK_EXIT(tDecodeFloat(pDecoder, &item->fval)); + TAOS_CHECK_EXIT(tDecodeFloat(pDecoder, &item.fval)); break; case CFG_DTYPE_STRING: case CFG_DTYPE_DIR: case CFG_DTYPE_LOCALE: case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: - TAOS_CHECK_EXIT(tDecodeCStr(pDecoder, &item->str)); + TAOS_CHECK_EXIT(tDecodeCStr(pDecoder, &item.str)); break; } } @@ -2803,4 +2806,78 @@ int32_t compareSConfigItemArrays(SArray *mArray, const SArray *dArray, SArray *d } return code; +} + +void printConfigNotMatch(SArray *array) { + uError( + "The global configuration parameters in the configuration file do not match those in the cluster. Please " + "turn off the forceReadConfigFile option or modify the global configuration parameters that are not " + "configured."); + int32_t sz = taosArrayGetSize(array); + for (int i = 0; i < sz; i++) { + SConfigItem *item = (SConfigItem *)taosArrayGet(array, i); + switch (item->dtype) { + { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + uError("config %s in cluster value is:%d", item->name, item->bval); + break; + case CFG_DTYPE_INT32: + uError("config %s in cluster value is:%d", item->name, item->i32); + break; + case CFG_DTYPE_INT64: + uError("config %s in cluster value is:%" PRId64, item->name, item->i64); + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + uError("config %s in cluster value is:%f", item->name, item->fval); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + uError("config %s in cluster value is:%s", item->name, item->str); + break; + } + } + } +} + +void printConfigNotMatch(SArray *array) { + uError( + "The global configuration parameters in the configuration file do not match those in the cluster. Please " + "turn off the forceReadConfigFile option or modify the global configuration parameters that are not " + "configured."); + int32_t sz = taosArrayGetSize(array); + for (int i = 0; i < sz; i++) { + SConfigItem *item = (SConfigItem *)taosArrayGet(array, i); + switch (item->dtype) { + { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + uError("config %s in cluster value is:%d", item->name, item->bval); + break; + case CFG_DTYPE_INT32: + uError("config %s in cluster value is:%d", item->name, item->i32); + break; + case CFG_DTYPE_INT64: + uError("config %s in cluster value is:%" PRId64, item->name, item->i64); + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + uError("config %s in cluster value is:%f", item->name, item->fval); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + uError("config %s in cluster value is:%s", item->name, item->str); + break; + } + } + } } \ No newline at end of file diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 9e94c92fac..c73101c850 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1653,6 +1653,7 @@ int32_t tSerializeSConfigRsp(void *buf, int32_t bufLen, SConfigRsp *pRsp) { TAOS_CHECK_EXIT(tEncodeI32(&encoder, pRsp->forceReadConfig)); TAOS_CHECK_EXIT(tEncodeI32(&encoder, pRsp->isConifgVerified)); TAOS_CHECK_EXIT(tEncodeI32(&encoder, pRsp->isVersionVerified)); + TAOS_CHECK_EXIT(tEncodeI32(&encoder, pRsp->cver)); if ((!pRsp->isConifgVerified) || (!pRsp->isVersionVerified)) { TAOS_CHECK_EXIT(tSerializeSConfigArray(&encoder, pRsp->array)); } @@ -1684,7 +1685,8 @@ _exit: tDecoderClear(&decoder); return code; } -void tFreeSConfigRsp(SConfigRsp *pRsp); + +void tFreeSConfigRsp(SConfigRsp *pRsp) { taosArrayDestroy(pRsp->array); } int32_t tSerializeSDnodeInfoReq(void *buf, int32_t bufLen, SDnodeInfoReq *pReq) { int32_t code = 0, lino = 0; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index a2aa16e2af..b677ae1bd6 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -283,6 +283,45 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { dmProcessStatusRsp(pMgmt, &rpcRsp); } +static void dmProcessConfigRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { + const STraceId *trace = &pRsp->info.traceId; + SConfigRsp configRsp = {0}; + dGTrace("status rsp received from mnode, statusSeq:%d code:0x%x", pMgmt->statusSeq, pRsp->code); + + if (pRsp->code != 0) { + if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pMgmt->pData->dropped && pMgmt->pData->dnodeId > 0) { + dGInfo("dnode:%d, set to dropped since not exist in mnode, statusSeq:%d", pMgmt->pData->dnodeId, + pMgmt->statusSeq); + pMgmt->pData->dropped = 1; + if (dmWriteEps(pMgmt->pData) != 0) { + dError("failed to write dnode file"); + } + dInfo("dnode will exit since it is in the dropped state"); + (void)raise(SIGINT); + } + } else { + if (pRsp->pCont != NULL && pRsp->contLen > 0 && + tDeserializeSConfigRsp(pRsp->pCont, pRsp->contLen, &configRsp) == 0) { + if (configRsp.forceReadConfig) { + if (configRsp.isConifgVerified) { + persistGlobalConfig(cfgGetGlobalCfg(tsCfg), pMgmt->path, configRsp.cver); + } else { + // log the difference configurations + printConfigNotMatch(configRsp.array); + } + goto _exit; + } + if (!configRsp.isVersionVerified) { + + persistGlobalConfig(cfgGetGlobalCfg(tsCfg), pMgmt->path, configRsp.cver); + } + } + } +_exit: + tFreeSConfigRsp(&configRsp); + rpcFreeCont(pRsp->pCont); +} + void dmSendConfigReq(SDnodeMgmt *pMgmt) { int32_t code = 0; SConfigReq req = {0}; @@ -327,10 +366,11 @@ void dmSendConfigReq(SDnodeMgmt *pMgmt) { dError("failed to send status req since %s", tstrerror(code)); return; } - if (rpcRsp.code != 0) { - } else { + dError("failed to send config req since %s", tstrerror(rpcRsp.code)); + return; } + dmProcessConfigRsp(pMgmt, &rpcRsp); } void dmUpdateStatusInfo(SDnodeMgmt *pMgmt) { diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index 27484459aa..8737d4cf9b 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -45,7 +45,7 @@ static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) { return code; } -static void mmBuildConfigForDeploy(SMnodeMgmt *pMgmt) { persistGlobalConfig(pMgmt->path, 0); } +static void mmBuildConfigForDeploy(SMnodeMgmt *pMgmt) { persistGlobalConfig(cfgGetGlobalCfg(tsCfg), pMgmt->path, 0); } static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInput, SMnodeOpt *pOption) { pOption->deploy = true; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 0cc02ec93c..9963f48e1a 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -934,6 +934,7 @@ static int32_t mndProcessConfigReq(SRpcMsg *pReq) { SArray *diffArray = taosArrayInit(16, sizeof(SConfigItem)); SConfigRsp configRsp = {0}; configRsp.forceReadConfig = configReq.forceReadConfig; + configRsp.cver = tsConfigVersion; if (configRsp.forceReadConfig) { // compare config array from configReq with current config array if (compareSConfigItemArrays(cfgGetGlobalCfg(tsCfg), configReq.array, diffArray)) { @@ -960,6 +961,7 @@ static int32_t mndProcessConfigReq(SRpcMsg *pReq) { goto _OVER; } _OVER: + mndReleaseDnode(pMnode, pDnode); return mndUpdClusterInfo(pReq); } From 281c1e67c24905bad15bc1df30419a2ee4f8ccc7 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 12 Nov 2024 09:48:24 +0800 Subject: [PATCH 14/99] fix merge error --- source/common/src/tglobal.c | 77 ++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 39 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 2da6b9eb2f..a4fd2f2985 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -660,9 +660,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorScope", tsRandErrScope, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "metaCacheMaxSize", tsMetaCacheMaxSize, -1, INT32_MAX, CFG_SCOPE_CLIENT, - CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); - + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "safetyCheckLevel", tsSafetyCheckLevel, 0, 5, CFG_SCOPE_BOTH, CFG_DYN_BOTH, + CFG_CATEGORY_GLOBAL)); tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH, CFG_DYN_BOTH, @@ -2845,39 +2844,39 @@ void printConfigNotMatch(SArray *array) { } } -void printConfigNotMatch(SArray *array) { - uError( - "The global configuration parameters in the configuration file do not match those in the cluster. Please " - "turn off the forceReadConfigFile option or modify the global configuration parameters that are not " - "configured."); - int32_t sz = taosArrayGetSize(array); - for (int i = 0; i < sz; i++) { - SConfigItem *item = (SConfigItem *)taosArrayGet(array, i); - switch (item->dtype) { - { - case CFG_DTYPE_NONE: - break; - case CFG_DTYPE_BOOL: - uError("config %s in cluster value is:%d", item->name, item->bval); - break; - case CFG_DTYPE_INT32: - uError("config %s in cluster value is:%d", item->name, item->i32); - break; - case CFG_DTYPE_INT64: - uError("config %s in cluster value is:%" PRId64, item->name, item->i64); - break; - case CFG_DTYPE_FLOAT: - case CFG_DTYPE_DOUBLE: - uError("config %s in cluster value is:%f", item->name, item->fval); - break; - case CFG_DTYPE_STRING: - case CFG_DTYPE_DIR: - case CFG_DTYPE_LOCALE: - case CFG_DTYPE_CHARSET: - case CFG_DTYPE_TIMEZONE: - uError("config %s in cluster value is:%s", item->name, item->str); - break; - } - } - } -} \ No newline at end of file +// void printConfigNotMatch(SArray *array) { +// uError( +// "The global configuration parameters in the configuration file do not match those in the cluster. Please " +// "turn off the forceReadConfigFile option or modify the global configuration parameters that are not " +// "configured."); +// int32_t sz = taosArrayGetSize(array); +// for (int i = 0; i < sz; i++) { +// SConfigItem *item = (SConfigItem *)taosArrayGet(array, i); +// switch (item->dtype) { +// { +// case CFG_DTYPE_NONE: +// break; +// case CFG_DTYPE_BOOL: +// uError("config %s in cluster value is:%d", item->name, item->bval); +// break; +// case CFG_DTYPE_INT32: +// uError("config %s in cluster value is:%d", item->name, item->i32); +// break; +// case CFG_DTYPE_INT64: +// uError("config %s in cluster value is:%" PRId64, item->name, item->i64); +// break; +// case CFG_DTYPE_FLOAT: +// case CFG_DTYPE_DOUBLE: +// uError("config %s in cluster value is:%f", item->name, item->fval); +// break; +// case CFG_DTYPE_STRING: +// case CFG_DTYPE_DIR: +// case CFG_DTYPE_LOCALE: +// case CFG_DTYPE_CHARSET: +// case CFG_DTYPE_TIMEZONE: +// uError("config %s in cluster value is:%s", item->name, item->str); +// break; +// } +// } +// } +// } \ No newline at end of file From 9758cd782578f016bc92e198783f69b0e38790a3 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 12 Nov 2024 13:40:19 +0800 Subject: [PATCH 15/99] fix some errors. --- source/common/src/tglobal.c | 8 +------- source/dnode/mgmt/mgmt_dnode/inc/dmInt.h | 1 + source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 4 ++-- source/dnode/mgmt/mgmt_dnode/src/dmInt.c | 4 ++++ source/dnode/mnode/impl/src/mndDnode.c | 2 ++ 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index a4fd2f2985..c8f0759eb1 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -660,8 +660,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorScope", tsRandErrScope, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "safetyCheckLevel", tsSafetyCheckLevel, 0, 5, CFG_SCOPE_BOTH, CFG_DYN_BOTH, - CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "safetyCheckLevel", tsSafetyCheckLevel, 0, 5, CFG_SCOPE_BOTH, CFG_DYN_BOTH,CFG_CATEGORY_GLOBAL)); tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH, CFG_DYN_BOTH, @@ -1948,11 +1947,6 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile TAOS_CHECK_GOTO(taosSetAllDebugFlag(tsCfg, pItem->i32), &lino, _exit); cfgDumpCfg(tsCfg, tsc, false); - if (!tsc) { - if ((code = persistLocalConfig(tsDataDir)) != 0) { - goto _exit; - } - } TAOS_CHECK_GOTO(taosCheckGlobalCfg(), &lino, _exit); _exit: diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index 47bbb88ee1..2108a097ee 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -66,6 +66,7 @@ int32_t dmProcessCreateEncryptKeyReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); // dmWorker.c int32_t dmPutNodeMsgToMgmtQueue(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmStartStatusThread(SDnodeMgmt *pMgmt); +int32_t dmStartConfigThread(SDnodeMgmt *pMgmt); int32_t dmStartStatusInfoThread(SDnodeMgmt *pMgmt); void dmStopStatusThread(SDnodeMgmt *pMgmt); void dmStopStatusInfoThread(SDnodeMgmt *pMgmt); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index b677ae1bd6..0b13f1041f 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -308,14 +308,14 @@ static void dmProcessConfigRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { } else { // log the difference configurations printConfigNotMatch(configRsp.array); + goto _exit; } - goto _exit; } if (!configRsp.isVersionVerified) { - persistGlobalConfig(cfgGetGlobalCfg(tsCfg), pMgmt->path, configRsp.cver); } } + persistLocalConfig(pMgmt->path); } _exit: tFreeSConfigRsp(&configRsp); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index 04b4e9101c..1b6fabad00 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -23,6 +23,10 @@ static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) { if ((code = dmStartStatusThread(pMgmt)) != 0) { return code; } + + if ((code = dmStartConfigThread(pMgmt)) != 0) { + return code; + } if ((code = dmStartStatusInfoThread(pMgmt)) != 0) { return code; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 9963f48e1a..060acd228c 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -960,6 +960,8 @@ static int32_t mndProcessConfigReq(SRpcMsg *pReq) { code = contLen; goto _OVER; } + pReq->info.rspLen = contLen; + pReq->info.rsp = pHead; _OVER: mndReleaseDnode(pMnode, pDnode); From 90a2dac7d646df7d04874b397ec8867abce0ea83 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 13 Nov 2024 13:56:39 +0800 Subject: [PATCH 16/99] Add decode for local/global.json --- include/common/tglobal.h | 4 +- source/common/src/tglobal.c | 180 +++++++++++++++----- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 3 + 3 files changed, 146 insertions(+), 41 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 89aee31b6a..4095b650f2 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -300,8 +300,8 @@ int32_t persistLocalConfig(const char *path); int32_t localConfigSerialize(SArray *array, char **serialized); int32_t tSerializeSConfigArray(SEncoder *pEncoder, SArray *array); int32_t tDeserializeSConfigArray(SDecoder *pDecoder, SArray *array); - -void printConfigNotMatch(SArray *array); +int32_t setAllConfigs(SConfig *pCfg); +void printConfigNotMatch(SArray *array); int32_t compareSConfigItemArrays(SArray *mArray, const SArray *dArray, SArray *diffArray); #ifdef __cplusplus diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c8f0759eb1..7462bfe1cd 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -660,7 +660,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorScope", tsRandErrScope, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "safetyCheckLevel", tsSafetyCheckLevel, 0, 5, CFG_SCOPE_BOTH, CFG_DYN_BOTH,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "safetyCheckLevel", tsSafetyCheckLevel, 0, 5, CFG_SCOPE_BOTH, CFG_DYN_BOTH, + CFG_CATEGORY_GLOBAL)); tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH, CFG_DYN_BOTH, @@ -1890,6 +1891,140 @@ static int32_t cfgInitWrapper(SConfig **pCfg) { TAOS_RETURN(TSDB_CODE_SUCCESS); } +int32_t setAllConfigs(SConfig *pCfg) { + int32_t code = 0; + int32_t lino = -1; + TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit); + TAOS_CHECK_GOTO(taosUpdateServerCfg(tsCfg), &lino, _exit); + TAOS_CHECK_GOTO(taosSetServerCfg(tsCfg), &lino, _exit); + TAOS_CHECK_GOTO(taosSetReleaseCfg(tsCfg), &lino, _exit); + TAOS_CHECK_GOTO(taosSetTfsCfg(tsCfg), &lino, _exit); + TAOS_CHECK_GOTO(taosSetS3Cfg(tsCfg), &lino, _exit); + TAOS_CHECK_GOTO(taosSetSystemCfg(tsCfg), &lino, _exit); + TAOS_CHECK_GOTO(taosSetFileHandlesLimit(), &lino, _exit); +_exit: + TAOS_RETURN(code); +} + +int32_t cfgDeserialize(SArray *array, char *buf, bool isGlobal) { + cJSON *pRoot = cJSON_Parse(buf); + if (pRoot == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (isGlobal) { + cJSON *pItem = cJSON_GetObjectItem(pRoot, "version"); + if (pItem == NULL) { + cJSON_Delete(pRoot); + return TSDB_CODE_OUT_OF_MEMORY; + } + tsConfigVersion = pItem->valueint; + } + + int32_t sz = taosArrayGetSize(array); + cJSON *configs = cJSON_GetObjectItem(pRoot, "configs"); + if (configs == NULL) { + cJSON_Delete(pRoot); + return TSDB_CODE_OUT_OF_MEMORY; + } + for (int i = 0; i < sz; i++) { + SConfigItem *item = (SConfigItem *)taosArrayGet(array, i); + cJSON *pItem = cJSON_GetObjectItem(configs, item->name); + if (pItem == NULL) { + continue; + } + switch (item->dtype) { + { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + item->bval = cJSON_IsTrue(pItem); + break; + case CFG_DTYPE_INT32: + item->i32 = pItem->valueint; + break; + case CFG_DTYPE_INT64: + item->i64 = atoll(cJSON_GetStringValue(pItem)); + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + item->fval = pItem->valuedouble; + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + tstrncpy(item->str, pItem->valuestring, strlen(pItem->valuestring)); + break; + } + } + } + cJSON_Delete(pRoot); + return TSDB_CODE_SUCCESS; +} + +int32_t readCfgFile(const char *path, bool isGlobal) { + int32_t code = 0; + char filename[CONFIG_FILE_LEN] = {0}; + SArray *array = NULL; + if (isGlobal) { + array = cfgGetGlobalCfg(tsCfg); + snprintf(filename, sizeof(filename), "%s%sconfig%sglobal.json", path, TD_DIRSEP, TD_DIRSEP); + } else { + array = cfgGetLocalCfg(tsCfg); + snprintf(filename, sizeof(filename), "%s%sconfig%slocal.json", path, TD_DIRSEP, TD_DIRSEP); + } + + int64_t fileSize = 0; + char *buf = NULL; + if (taosStatFile(filename, &fileSize, NULL, NULL) != 0) { + if (terrno != ENOENT) { + uError("failed to stat file:%s , since %s", filename, tstrerror(code)); + code = terrno; + } + TAOS_RETURN(TSDB_CODE_SUCCESS); + } + TdFilePtr pFile = taosOpenFile(filename, TD_FILE_READ); + if (pFile == NULL) { + code = terrno; + TAOS_RETURN(code); + } + buf = (char *)taosMemoryMalloc(fileSize + 1); + if (taosReadFile(pFile, buf, fileSize) != fileSize) { + uError("failed to read file:%s , config since %s", filename, tstrerror(code)); + (void)taosCloseFile(&pFile); + taosMemoryFree(buf); + TAOS_RETURN(terrno); + } + char *serialized = NULL; + code = cfgDeserialize(array, buf, isGlobal); + if (code != TSDB_CODE_SUCCESS) { + uError("failed to deserialize config from %s since %s", filename, tstrerror(code)); + TAOS_RETURN(code); + } + TAOS_RETURN(code); +} + +int32_t tryLoadCfgFromDataDir(SConfig *pCfg) { + int32_t code = 0; + SConfigItem *pItem = NULL; + TAOS_CHECK_GET_CFG_ITEM(tsCfg, pItem, "forceReadConfig"); + tsForceReadConfig = pItem->i32; + if (!tsForceReadConfig) { + code = readCfgFile(tsDataDir, false); + if (code != TSDB_CODE_SUCCESS) { + uError("failed to read local config from %s since %s", tsDataDir, tstrerror(code)); + TAOS_RETURN(code); + } + code = readCfgFile(tsDataDir, true); + if (code != TSDB_CODE_SUCCESS) { + uError("failed to read global config from %s since %s", tsDataDir, tstrerror(code)); + TAOS_RETURN(code); + } + } + TAOS_RETURN(code); +} + int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { if (tsCfg != NULL) TAOS_RETURN(TSDB_CODE_SUCCESS); @@ -1924,6 +2059,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile tsCfg = NULL; TAOS_RETURN(code); } + tryLoadCfgFromDataDir(tsCfg); if (tsc) { TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit); @@ -2602,6 +2738,7 @@ int32_t localConfigSerialize(SArray *array, char **serialized) { return TSDB_CODE_SUCCESS; } +// TODO:close file when error int32_t persistGlobalConfig(SArray *array, const char *path, int32_t version) { // TODO: just tmp ,refactor later int32_t code = 0; @@ -2634,6 +2771,7 @@ int32_t persistGlobalConfig(SArray *array, const char *path, int32_t version) { TAOS_RETURN(code); } taosWriteFile(pConfigFile, serialized, strlen(serialized)); + (void)taosCloseFile(&pConfigFile); return TSDB_CODE_SUCCESS; } @@ -2669,6 +2807,7 @@ int32_t persistLocalConfig(const char *path) { TAOS_RETURN(code); } taosWriteFile(pConfigFile, serialized, strlen(serialized)); + (void)taosCloseFile(&pConfigFile); return TSDB_CODE_SUCCESS; } @@ -2836,41 +2975,4 @@ void printConfigNotMatch(SArray *array) { } } } -} - -// void printConfigNotMatch(SArray *array) { -// uError( -// "The global configuration parameters in the configuration file do not match those in the cluster. Please " -// "turn off the forceReadConfigFile option or modify the global configuration parameters that are not " -// "configured."); -// int32_t sz = taosArrayGetSize(array); -// for (int i = 0; i < sz; i++) { -// SConfigItem *item = (SConfigItem *)taosArrayGet(array, i); -// switch (item->dtype) { -// { -// case CFG_DTYPE_NONE: -// break; -// case CFG_DTYPE_BOOL: -// uError("config %s in cluster value is:%d", item->name, item->bval); -// break; -// case CFG_DTYPE_INT32: -// uError("config %s in cluster value is:%d", item->name, item->i32); -// break; -// case CFG_DTYPE_INT64: -// uError("config %s in cluster value is:%" PRId64, item->name, item->i64); -// break; -// case CFG_DTYPE_FLOAT: -// case CFG_DTYPE_DOUBLE: -// uError("config %s in cluster value is:%f", item->name, item->fval); -// break; -// case CFG_DTYPE_STRING: -// case CFG_DTYPE_DIR: -// case CFG_DTYPE_LOCALE: -// case CFG_DTYPE_CHARSET: -// case CFG_DTYPE_TIMEZONE: -// uError("config %s in cluster value is:%s", item->name, item->str); -// break; -// } -// } -// } -// } \ No newline at end of file +} \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 0b13f1041f..8f72758cd9 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -312,10 +312,13 @@ static void dmProcessConfigRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { } } if (!configRsp.isVersionVerified) { + cfgLoadFromArray(tsCfg, configRsp.array); persistGlobalConfig(cfgGetGlobalCfg(tsCfg), pMgmt->path, configRsp.cver); } } + setAllConfigs(tsCfg); persistLocalConfig(pMgmt->path); + setAllConfigs(tsCfg); } _exit: tFreeSConfigRsp(&configRsp); From 748bb000b0cf60cbb2966a17fd83bf572b0d1f3d Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 13 Nov 2024 16:51:26 +0800 Subject: [PATCH 17/99] fix some error and rename some functions --- include/common/tglobal.h | 3 ++- include/util/tconfig.h | 7 +++++-- source/common/src/tglobal.c | 23 +++++++++++---------- source/common/src/tmsg.c | 3 +++ source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 11 +++++----- source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 4 +++- source/dnode/mnode/impl/src/mndDnode.c | 12 +++++------ source/util/src/tconfig.c | 6 ++++-- 8 files changed, 40 insertions(+), 29 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 4095b650f2..5ebfd24d5e 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -53,7 +53,8 @@ extern char tsVersionName[]; extern uint16_t tsServerPort; extern int32_t tsVersion; extern int32_t tsForceReadConfig; -extern int32_t tsConfigVersion; +extern int32_t tsdmConfigVersion; +extern int32_t tsmmConfigVersion; extern int32_t tsConfigInited; extern int32_t tsStatusInterval; extern int32_t tsNumOfSupportVnodes; diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 197bf04c3c..98e0df9655 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -148,8 +148,11 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump); void cfgDumpCfgS3(SConfig *pCfg, bool tsc, bool dump); int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl); -SArray *cfgGetLocalCfg(SConfig *pCfg); -SArray *cfgGetGlobalCfg(SConfig *pCfg); +SArray *getLocalCfg(SConfig *pCfg); +SArray *getGlobalCfg(SConfig *pCfg); + +void setLocalCfg(SConfig *pCfg, SArray *pArray); +void setGlobalCfg(SConfig *pCfg, SArray *pArray); #ifdef __cplusplus } #endif diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 7462bfe1cd..c8b214ecfa 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -45,7 +45,8 @@ char tsVersionName[16] = "community"; uint16_t tsServerPort = 6030; int32_t tsVersion = 30000000; int32_t tsForceReadConfig = 0; -int32_t tsConfigVersion = 0; +int32_t tsdmConfigVersion = -1; +int32_t tsmmConfigVersion = 0; int32_t tsConfigInited = 0; int32_t tsStatusInterval = 1; // second int32_t tsNumOfSupportVnodes = 256; @@ -1917,7 +1918,7 @@ int32_t cfgDeserialize(SArray *array, char *buf, bool isGlobal) { cJSON_Delete(pRoot); return TSDB_CODE_OUT_OF_MEMORY; } - tsConfigVersion = pItem->valueint; + tsdmConfigVersion = pItem->valueint; } int32_t sz = taosArrayGetSize(array); @@ -1968,10 +1969,10 @@ int32_t readCfgFile(const char *path, bool isGlobal) { char filename[CONFIG_FILE_LEN] = {0}; SArray *array = NULL; if (isGlobal) { - array = cfgGetGlobalCfg(tsCfg); + array = getGlobalCfg(tsCfg); snprintf(filename, sizeof(filename), "%s%sconfig%sglobal.json", path, TD_DIRSEP, TD_DIRSEP); } else { - array = cfgGetLocalCfg(tsCfg); + array = getLocalCfg(tsCfg); snprintf(filename, sizeof(filename), "%s%sconfig%slocal.json", path, TD_DIRSEP, TD_DIRSEP); } @@ -2801,7 +2802,7 @@ int32_t persistLocalConfig(const char *path) { TAOS_RETURN(code); } char *serialized = NULL; - code = localConfigSerialize(cfgGetLocalCfg(tsCfg), &serialized); + code = localConfigSerialize(getLocalCfg(tsCfg), &serialized); if (code != TSDB_CODE_SUCCESS) { uError("failed to serialize local config since %s", tstrerror(code)); TAOS_RETURN(code); @@ -2852,16 +2853,15 @@ _exit: } int32_t tDeserializeSConfigArray(SDecoder *pDecoder, SArray *array) { - int32_t code = 0; - int32_t lino = 0; - int32_t sz = 0; - ECfgDataType dtype = CFG_DTYPE_NONE; + int32_t code = 0; + int32_t lino = 0; + int32_t sz = 0; tDecodeI32(pDecoder, &sz); for (int i = 0; i < sz; i++) { SConfigItem item = {0}; TAOS_CHECK_EXIT(tDecodeCStr(pDecoder, &item.name)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, (int32_t *)&dtype)); - switch (dtype) { + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, (int32_t *)&item.dtype)); + switch (item.dtype) { { case CFG_DTYPE_NONE: break; @@ -2887,6 +2887,7 @@ int32_t tDeserializeSConfigArray(SDecoder *pDecoder, SArray *array) { break; } } + taosArrayPush(array, &item); } _exit: return code; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index c73101c850..83577403aa 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1635,6 +1635,7 @@ int32_t tDeserializeSConfigReq(void *buf, int32_t bufLen, SConfigReq *pReq) { TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->cver)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->forceReadConfig)); if (pReq->forceReadConfig) { + pReq->array = taosArrayInit(128, sizeof(SConfigItem)); TAOS_CHECK_EXIT(tDeserializeSConfigArray(&decoder, pReq->array)); } tEndDecode(&decoder); @@ -1677,7 +1678,9 @@ int32_t tDeserializeSConfigRsp(void *buf, int32_t bufLen, SConfigRsp *pRsp) { TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pRsp->forceReadConfig)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pRsp->isConifgVerified)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pRsp->isVersionVerified)); + TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pRsp->cver)); if ((!pRsp->isConifgVerified) || (!pRsp->isVersionVerified)) { + pRsp->array = taosArrayInit(128, sizeof(SConfigItem)); TAOS_CHECK_EXIT(tDeserializeSConfigArray(&decoder, pRsp->array)); } _exit: diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 8f72758cd9..15176f58a4 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -304,7 +304,7 @@ static void dmProcessConfigRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { tDeserializeSConfigRsp(pRsp->pCont, pRsp->contLen, &configRsp) == 0) { if (configRsp.forceReadConfig) { if (configRsp.isConifgVerified) { - persistGlobalConfig(cfgGetGlobalCfg(tsCfg), pMgmt->path, configRsp.cver); + persistGlobalConfig(getGlobalCfg(tsCfg), pMgmt->path, configRsp.cver); } else { // log the difference configurations printConfigNotMatch(configRsp.array); @@ -312,13 +312,12 @@ static void dmProcessConfigRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { } } if (!configRsp.isVersionVerified) { - cfgLoadFromArray(tsCfg, configRsp.array); - persistGlobalConfig(cfgGetGlobalCfg(tsCfg), pMgmt->path, configRsp.cver); + persistGlobalConfig(configRsp.array, pMgmt->path, configRsp.cver); } } setAllConfigs(tsCfg); persistLocalConfig(pMgmt->path); - setAllConfigs(tsCfg); + tsConfigInited = 1; } _exit: tFreeSConfigRsp(&configRsp); @@ -329,9 +328,9 @@ void dmSendConfigReq(SDnodeMgmt *pMgmt) { int32_t code = 0; SConfigReq req = {0}; - req.cver = tsConfigVersion; + req.cver = tsdmConfigVersion; req.forceReadConfig = tsForceReadConfig; - req.array = cfgGetGlobalCfg(tsCfg); + req.array = getGlobalCfg(tsCfg); dDebug("send config req to mnode, configVersion:%d", req.cver); int32_t contLen = tSerializeSConfigReq(NULL, 0, &req); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index 8737d4cf9b..b3b740f0fb 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -45,7 +45,9 @@ static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) { return code; } -static void mmBuildConfigForDeploy(SMnodeMgmt *pMgmt) { persistGlobalConfig(cfgGetGlobalCfg(tsCfg), pMgmt->path, 0); } +static void mmBuildConfigForDeploy(SMnodeMgmt *pMgmt) { + persistGlobalConfig(getGlobalCfg(tsCfg), pMgmt->path, tsmmConfigVersion); +} static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInput, SMnodeOpt *pOption) { pOption->deploy = true; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 060acd228c..d981831c12 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -934,21 +934,21 @@ static int32_t mndProcessConfigReq(SRpcMsg *pReq) { SArray *diffArray = taosArrayInit(16, sizeof(SConfigItem)); SConfigRsp configRsp = {0}; configRsp.forceReadConfig = configReq.forceReadConfig; - configRsp.cver = tsConfigVersion; + configRsp.cver = tsmmConfigVersion; if (configRsp.forceReadConfig) { // compare config array from configReq with current config array - if (compareSConfigItemArrays(cfgGetGlobalCfg(tsCfg), configReq.array, diffArray)) { + if (compareSConfigItemArrays(getGlobalCfg(tsCfg), configReq.array, diffArray)) { configRsp.array = diffArray; } else { configRsp.isConifgVerified = 1; taosArrayDestroy(diffArray); } } else { - configRsp.array = cfgGetGlobalCfg(tsCfg); - if (configReq.cver == tsConfigVersion) { - configRsp.isConifgVerified = 1; + configRsp.array = getGlobalCfg(tsCfg); + if (configReq.cver == tsmmConfigVersion) { + configRsp.isVersionVerified = 1; } else { - configRsp.array = cfgGetGlobalCfg(tsCfg); + configRsp.array = getGlobalCfg(tsCfg); } } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 3bac376242..3fed23e8a2 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -1537,5 +1537,7 @@ void cfgDestroyIter(SConfigIter *pIter) { taosMemoryFree(pIter); } -SArray *cfgGetLocalCfg(SConfig *pCfg) { return pCfg->localArray; } -SArray *cfgGetGlobalCfg(SConfig *pCfg) { return pCfg->globalArray; } \ No newline at end of file +SArray *getLocalCfg(SConfig *pCfg) { return pCfg->localArray; } +SArray *getGlobalCfg(SConfig *pCfg) { return pCfg->globalArray; } +void setLocalCfg(SConfig *pCfg, SArray *pArray) { pCfg->localArray = pArray; }; +void setGlobalCfg(SConfig *pCfg, SArray *pArray) { pCfg->globalArray = pArray; }; \ No newline at end of file From cc3f4504bf85e485711135e72f8e7cf461514899 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 14 Nov 2024 17:20:50 +0800 Subject: [PATCH 18/99] Add mnd init cfg and mnd cfg trans. --- include/util/tconfig.h | 1 + source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 2 +- source/dnode/mnode/impl/inc/mndConfig.h | 36 +++ source/dnode/mnode/impl/inc/mndDef.h | 27 +- source/dnode/mnode/impl/src/mndConfig.c | 305 ++++++++++++++++++++ source/dnode/mnode/impl/src/mndDb.c | 24 +- source/dnode/mnode/impl/src/mndDnode.c | 44 --- source/dnode/mnode/impl/src/mndMain.c | 4 +- source/dnode/mnode/sdb/inc/sdb.h | 9 +- source/dnode/mnode/sdb/src/sdbRaw.c | 23 ++ source/util/src/tconfig.c | 41 +++ 11 files changed, 444 insertions(+), 72 deletions(-) create mode 100644 source/dnode/mnode/impl/inc/mndConfig.h create mode 100644 source/dnode/mnode/impl/src/mndConfig.c diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 98e0df9655..312a10e2d6 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -118,6 +118,7 @@ void cfgCleanup(SConfig *pCfg); int32_t cfgGetSize(SConfig *pCfg); SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName); int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype, bool lock); +int32_t cfgUpdateItem(SConfigItem *pItem, SConfigItem *newItem); int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer); int32_t cfgCreateIter(SConfig *pConf, SConfigIter **ppIter); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 7b01609c29..8e595f76c9 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -200,7 +200,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_CONN, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_NOTIFY, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/inc/mndConfig.h b/source/dnode/mnode/impl/inc/mndConfig.h new file mode 100644 index 0000000000..22a115b30b --- /dev/null +++ b/source/dnode/mnode/impl/inc/mndConfig.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_MND_CONFIG_H_ +#define _TD_MND_CONFIG_H_ + +#include "mndInt.h" + +#ifdef __cplusplus +extern "C" { +#endif +int32_t mndInitConfig(SMnode *pMnode); +SSdbRaw *mnCfgActionEncode(SConfigItem *pCfg); +SSdbRow *mndCfgActionDecode(SSdbRaw *pRaw); +static int32_t mndCfgActionInsert(SSdb *pSdb, SConfigItem *item); +static int32_t mndCfgActionDelete(SSdb *pSdb, SConfigItem *item); +static int32_t mndCfgActionUpdate(SSdb *pSdb, SConfigItem *oldItem, SConfigItem *newItem); + +static int32_t mndProcessConfigReq(SRpcMsg *pReq); +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_ARBGROUP_H_*/ diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index d2d9b2e8eb..c2ee3944e5 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -106,8 +106,8 @@ typedef enum { TRN_CONFLICT_GLOBAL = 1, TRN_CONFLICT_DB = 2, TRN_CONFLICT_DB_INSIDE = 3, -// TRN_CONFLICT_TOPIC = 4, -// TRN_CONFLICT_TOPIC_INSIDE = 5, + // TRN_CONFLICT_TOPIC = 4, + // TRN_CONFLICT_TOPIC_INSIDE = 5, TRN_CONFLICT_ARBGROUP = 6, TRN_CONFLICT_TSMA = 7, } ETrnConflct; @@ -316,6 +316,7 @@ typedef struct { TdThreadMutex mutex; } SArbGroup; + typedef struct { int32_t maxUsers; int32_t maxDbs; @@ -649,12 +650,12 @@ typedef struct { int32_t maxPollIntervalMs; } SMqConsumerObj; -int32_t tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType, - char *topic, SCMSubscribeReq *subscribe, SMqConsumerObj** ppConsumer); -void tClearSMqConsumerObj(SMqConsumerObj* pConsumer); -void tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer); -int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer); -void* tDecodeSMqConsumerObj(const void* buf, SMqConsumerObj* pConsumer, int8_t sver); +int32_t tNewSMqConsumerObj(int64_t consumerId, char* cgroup, int8_t updateType, char* topic, SCMSubscribeReq* subscribe, + SMqConsumerObj** ppConsumer); +void tClearSMqConsumerObj(SMqConsumerObj* pConsumer); +void tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer); +int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer); +void* tDecodeSMqConsumerObj(const void* buf, SMqConsumerObj* pConsumer, int8_t sver); typedef struct { int32_t vgId; @@ -693,11 +694,11 @@ typedef struct { char* qmsg; // SubPlanToString } SMqSubscribeObj; -int32_t tNewSubscribeObj(const char *key, SMqSubscribeObj **ppSub); -int32_t tCloneSubscribeObj(const SMqSubscribeObj* pSub, SMqSubscribeObj **ppSub); -void tDeleteSubscribeObj(SMqSubscribeObj* pSub); -int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub); -void* tDecodeSubscribeObj(const void* buf, SMqSubscribeObj* pSub, int8_t sver); +int32_t tNewSubscribeObj(const char* key, SMqSubscribeObj** ppSub); +int32_t tCloneSubscribeObj(const SMqSubscribeObj* pSub, SMqSubscribeObj** ppSub); +void tDeleteSubscribeObj(SMqSubscribeObj* pSub); +int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub); +void* tDecodeSubscribeObj(const void* buf, SMqSubscribeObj* pSub, int8_t sver); // typedef struct { // int32_t epoch; diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c new file mode 100644 index 0000000000..59fdc6260a --- /dev/null +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -0,0 +1,305 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndConfig.h" +#include "mndDnode.h" +#include "mndTrans.h" + +#define CFG_VER_NUMBER 1 +#define CFG_RESERVE_SIZE 63 + +static int32_t mndProcessConfigReq(SRpcMsg *pReq); +static int32_t mndInitWriteCfg(SMnode *pMnode); +static int32_t mndInitReadCfg(SMnode *pMnode); + +int32_t mndSetCreateConfigCommitLogs(STrans *pTrans, SConfigItem *item); + +int32_t mndInitConfig(SMnode *pMnode) { + int32_t code = 0; + SSdbTable table = { + .sdbType = SDB_CFG, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mnCfgActionEncode, + .decodeFp = (SdbDecodeFp)mndCfgActionDecode, + .insertFp = (SdbInsertFp)mndCfgActionInsert, + .updateFp = (SdbUpdateFp)mndCfgActionUpdate, + .deleteFp = (SdbDeleteFp)mndCfgActionDelete, + }; + + if (pMnode->deploy) { + mndInitWriteCfg(pMnode); + } else { + mndInitReadCfg(pMnode); + } + + mndSetMsgHandle(pMnode, TDMT_MND_CONFIG, mndProcessConfigReq); + + return sdbSetTable(pMnode->pSdb, table); +} + +SSdbRaw *mnCfgActionEncode(SConfigItem *pCfg) { + int32_t code = 0; + int32_t lino = 0; + terrno = TSDB_CODE_OUT_OF_MEMORY; + char buf[30]; + + int32_t size = sizeof(SConfigItem) + CFG_RESERVE_SIZE; + SSdbRaw *pRaw = sdbAllocRaw(SDB_CFG, CFG_VER_NUMBER, size); + if (pRaw == NULL) goto _OVER; + + int32_t dataPos = 0; + SConfigItem *item = NULL; + SDB_SET_INT32(pRaw, dataPos, strlen(item->name), _OVER) + SDB_SET_BINARY(pRaw, dataPos, item->name, strlen(item->name), _OVER) + SDB_SET_INT32(pRaw, dataPos, item->dtype, _OVER) + switch (item->dtype) { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + SDB_SET_BOOL(pRaw, dataPos, item->bval, _OVER) + break; + case CFG_DTYPE_INT32: + SDB_SET_INT32(pRaw, dataPos, item->i32, _OVER); + break; + case CFG_DTYPE_INT64: + SDB_SET_INT64(pRaw, dataPos, item->i64, _OVER); + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + (void)sprintf(buf, "%f", item->fval); + SDB_SET_INT32(pRaw, dataPos, strlen(buf), _OVER) + SDB_SET_BINARY(pRaw, dataPos, buf, strlen(buf), _OVER) + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + SDB_SET_INT32(pRaw, dataPos, strlen(item->str), _OVER) + SDB_SET_BINARY(pRaw, dataPos, item->str, strlen(item->str), _OVER) + break; + } + + terrno = 0; + +_OVER: + if (terrno != 0) { + mError("cfg failed to encode to raw:%p since %s", pRaw, terrstr()); + sdbFreeRaw(pRaw); + return NULL; + } + mTrace("cfg encode to raw:%p, row:%p", pRaw, pCfg); + return pRaw; +} + +SSdbRow *mndCfgActionDecode(SSdbRaw *pRaw) { + int32_t code = 0; + int32_t lino = 0; + int32_t len = -1; + terrno = TSDB_CODE_OUT_OF_MEMORY; + SSdbRow *pRow = NULL; + SConfigItem *item = NULL; + + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; + + if (sver != CFG_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + goto _OVER; + } + + pRow = sdbAllocRow(sizeof(SSdbRaw)); + if (pRow == NULL) goto _OVER; + + item = sdbGetRowObj(pRow); + if (item == NULL) goto _OVER; + int32_t dataPos = 0; + SDB_GET_INT32(pRaw, dataPos, &len, _OVER) + SDB_GET_BINARY(pRaw, dataPos, item->name, len, _OVER) + SDB_GET_INT32(pRaw, dataPos, (int32_t *)&item->dtype, _OVER) + switch (item->dtype) { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + SDB_GET_BOOL(pRaw, dataPos, &item->bval, _OVER) + break; + case CFG_DTYPE_INT32: + SDB_GET_INT32(pRaw, dataPos, &item->i32, _OVER); + break; + case CFG_DTYPE_INT64: + SDB_GET_INT64(pRaw, dataPos, &item->i64, _OVER); + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + SDB_GET_INT32(pRaw, dataPos, &len, _OVER) + char *buf = taosMemoryMalloc(len + 1); + SDB_GET_BINARY(pRaw, dataPos, buf, len, _OVER) + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + SDB_GET_INT32(pRaw, dataPos, &len, _OVER) + SDB_GET_BINARY(pRaw, dataPos, item->str, len, _OVER) + break; + } + +_OVER: + if (terrno != 0) { + mError("cfg failed to decode from raw:%p since %s", pRaw, terrstr()); + taosMemoryFreeClear(pRow); + return NULL; + } + + mTrace("cfg decode from raw:%p, row:%p", pRaw, item); + return pRow; +} + +static int32_t mndCfgActionInsert(SSdb *pSdb, SConfigItem *item) { + mTrace("cfg:%s, perform insert action, row:%p", item->name, item); + return 0; +} + +static int32_t mndCfgActionDelete(SSdb *pSdb, SConfigItem *item) { + mTrace("cfg:%s, perform delete action, row:%p", item->name, item); + return 0; +} + +static int32_t mndCfgActionUpdate(SSdb *pSdb, SConfigItem *pOld, SConfigItem *pNew) { + mTrace("cfg:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); + return 0; +} + +static int32_t mndProcessConfigReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SConfigReq configReq = {0}; + SDnodeObj *pDnode = NULL; + int32_t code = -1; + + tDeserializeSConfigReq(pReq->pCont, pReq->contLen, &configReq); + SArray *diffArray = taosArrayInit(16, sizeof(SConfigItem)); + SConfigRsp configRsp = {0}; + configRsp.forceReadConfig = configReq.forceReadConfig; + configRsp.cver = tsmmConfigVersion; + if (configRsp.forceReadConfig) { + // compare config array from configReq with current config array + if (compareSConfigItemArrays(getGlobalCfg(tsCfg), configReq.array, diffArray)) { + configRsp.array = diffArray; + } else { + configRsp.isConifgVerified = 1; + taosArrayDestroy(diffArray); + } + } else { + configRsp.array = getGlobalCfg(tsCfg); + if (configReq.cver == tsmmConfigVersion) { + configRsp.isVersionVerified = 1; + } else { + configRsp.array = getGlobalCfg(tsCfg); + } + } + + int32_t contLen = tSerializeSConfigRsp(NULL, 0, &configRsp); + void *pHead = rpcMallocCont(contLen); + contLen = tSerializeSConfigRsp(pHead, contLen, &configRsp); + taosArrayDestroy(diffArray); + if (contLen < 0) { + code = contLen; + goto _OVER; + } + pReq->info.rspLen = contLen; + pReq->info.rsp = pHead; +_OVER: + + mndReleaseDnode(pMnode, pDnode); + return TSDB_CODE_SUCCESS; +} + +int32_t mndInitWriteCfg(SMnode *pMnode) { + int code = -1; + size_t sz = 0; + + SConfigItem item = {0}; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "init-write-config"); + if (pTrans == NULL) { + mError("failed to init write cfg in create trans, since %s", terrstr()); + goto _OVER; + } + + // encode mnd config version + item = (SConfigItem){.name = "tsmmConfigVersion", .dtype = CFG_DTYPE_INT32, .i32 = tsmmConfigVersion}; + if ((code = mndSetCreateConfigCommitLogs(pTrans, &item)) != 0) { + mError("failed to init mnd config version, since %s", terrstr()); + } + sz = taosArrayGetSize(getGlobalCfg(tsCfg)); + + for (int i = 0; i < sz; ++i) { + SConfigItem *item = taosArrayGet(getGlobalCfg(tsCfg), i); + if ((code = mndSetCreateConfigCommitLogs(pTrans, item)) != 0) { + mError("failed to init mnd config:%s, since %s", item->name, terrstr()); + } + } + if ((code = mndTransCheckConflict(pMnode, pTrans)) != 0) goto _OVER; + if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER; + +_OVER: + mndTransDrop(pTrans); + return TSDB_CODE_SUCCESS; +} + +int32_t mndInitReadCfg(SMnode *pMnode) { + int32_t code = 0; + int32_t sz = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "init-read-config"); + if (pTrans == NULL) { + mError("failed to init read cfg in create trans, since %s", terrstr()); + goto _OVER; + } + SConfigItem *item = sdbAcquire(pMnode->pSdb, SDB_CFG, "tsmmConfigVersion"); + if (item == NULL) { + mInfo("failed to acquire mnd config version, since %s", terrstr()); + goto _OVER; + } else { + tsmmConfigVersion = item->i32; + sdbRelease(pMnode->pSdb, item); + } + + for (int i = 0; i < sz; ++i) { + SConfigItem *item = taosArrayGet(getGlobalCfg(tsCfg), i); + SConfigItem *newItem = sdbAcquire(pMnode->pSdb, SDB_CFG, item->name); + if (newItem == NULL) { + mInfo("failed to acquire mnd config:%s, since %s", item->name, terrstr()); + continue; + } + cfgUpdateItem(item, newItem); + sdbRelease(pMnode->pSdb, newItem); + } +_OVER: + mndTransDrop(pTrans); + return TSDB_CODE_SUCCESS; +} + +int32_t mndSetCreateConfigCommitLogs(STrans *pTrans, SConfigItem *item) { + int32_t code = 0; + SSdbRaw *pCommitRaw = mnCfgActionEncode(item); + if (pCommitRaw == NULL) { + code = terrno; + TAOS_RETURN(code); + } + if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw) != 0)) TAOS_RETURN(code); + if ((code = sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)) != 0) TAOS_RETURN(code); + return TSDB_CODE_SUCCESS; +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 0d17ccd0b0..6f6a45d374 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -14,10 +14,12 @@ */ #define _DEFAULT_SOURCE -#include "mndDb.h" #include "audit.h" +#include "command.h" #include "mndArbGroup.h" #include "mndCluster.h" +#include "mndConfig.h" +#include "mndDb.h" #include "mndDnode.h" #include "mndIndex.h" #include "mndPrivilege.h" @@ -34,7 +36,6 @@ #include "systable.h" #include "thttp.h" #include "tjson.h" -#include "command.h" #define DB_VER_NUMBER 1 #define DB_RESERVE_SIZE 27 @@ -1560,8 +1561,8 @@ static int32_t mndBuildDropVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj * static int32_t mndSetDropDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { int32_t code = 0; - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; while (1) { SVgObj *pVgroup = NULL; @@ -1941,9 +1942,9 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs, continue; } else { mTrace("db:%s, valid dbinfo, vgVersion:%d cfgVersion:%d stateTs:%" PRId64 - " numOfTables:%d, changed to vgVersion:%d cfgVersion:%d stateTs:%" PRId64 " numOfTables:%d", - pDbCacheInfo->dbFName, pDbCacheInfo->vgVersion, pDbCacheInfo->cfgVersion, pDbCacheInfo->stateTs, - pDbCacheInfo->numOfTable, pDb->vgVersion, pDb->cfgVersion, pDb->stateTs, numOfTable); + " numOfTables:%d, changed to vgVersion:%d cfgVersion:%d stateTs:%" PRId64 " numOfTables:%d", + pDbCacheInfo->dbFName, pDbCacheInfo->vgVersion, pDbCacheInfo->cfgVersion, pDbCacheInfo->stateTs, + pDbCacheInfo->numOfTable, pDb->vgVersion, pDb->cfgVersion, pDb->stateTs, numOfTable); } if (pDbCacheInfo->cfgVersion < pDb->cfgVersion) { @@ -1955,7 +1956,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs, rsp.pTsmaRsp = taosMemoryCalloc(1, sizeof(STableTSMAInfoRsp)); if (rsp.pTsmaRsp) rsp.pTsmaRsp->pTsmas = taosArrayInit(4, POINTER_BYTES); if (rsp.pTsmaRsp && rsp.pTsmaRsp->pTsmas) { - bool exist = false; + bool exist = false; int32_t code = mndGetDbTsmas(pMnode, 0, pDb->uid, rsp.pTsmaRsp, &exist); if (TSDB_CODE_SUCCESS != code) { mndReleaseDb(pMnode, pDb); @@ -2386,7 +2387,8 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, (const char *)strictVstr, false), &lino, _OVER); char durationVstr[128] = {0}; - int32_t len = formatDurationOrKeep(&durationVstr[VARSTR_HEADER_SIZE], sizeof(durationVstr) - VARSTR_HEADER_SIZE, pDb->cfg.daysPerFile); + int32_t len = formatDurationOrKeep(&durationVstr[VARSTR_HEADER_SIZE], sizeof(durationVstr) - VARSTR_HEADER_SIZE, + pDb->cfg.daysPerFile); varDataSetLen(durationVstr, len); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -2402,9 +2404,9 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, int32_t lenKeep2 = formatDurationOrKeep(keep2Str, sizeof(keep2Str), pDb->cfg.daysToKeep2); if (pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep1 || pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep2) { - len = sprintf(&keepVstr[VARSTR_HEADER_SIZE], "%s,%s,%s", keep1Str, keep2Str, keep0Str); + len = sprintf(&keepVstr[VARSTR_HEADER_SIZE], "%s,%s,%s", keep1Str, keep2Str, keep0Str); } else { - len = sprintf(&keepVstr[VARSTR_HEADER_SIZE], "%s,%s,%s", keep0Str, keep1Str, keep2Str); + len = sprintf(&keepVstr[VARSTR_HEADER_SIZE], "%s,%s,%s", keep0Str, keep1Str, keep2Str); } varDataSetLen(keepVstr, len); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index d981831c12..b4036e328e 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -84,7 +84,6 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq); static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq); static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessStatusReq(SRpcMsg *pReq); -static int32_t mndProcessConfigReq(SRpcMsg *pReq); static int32_t mndProcessNotifyReq(SRpcMsg *pReq); static int32_t mndProcessRestoreDnodeReq(SRpcMsg *pReq); static int32_t mndProcessStatisReq(SRpcMsg *pReq); @@ -123,7 +122,6 @@ int32_t mndInitDnode(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_CONFIG_DNODE, mndProcessConfigDnodeReq); mndSetMsgHandle(pMnode, TDMT_DND_CONFIG_DNODE_RSP, mndProcessConfigDnodeRsp); mndSetMsgHandle(pMnode, TDMT_MND_STATUS, mndProcessStatusReq); - mndSetMsgHandle(pMnode, TDMT_MND_CONFIG, mndProcessConfigReq); mndSetMsgHandle(pMnode, TDMT_MND_NOTIFY, mndProcessNotifyReq); mndSetMsgHandle(pMnode, TDMT_MND_DNODE_LIST, mndProcessDnodeListReq); mndSetMsgHandle(pMnode, TDMT_MND_SHOW_VARIABLES, mndProcessShowVariablesReq); @@ -924,49 +922,7 @@ _OVER: return mndUpdClusterInfo(pReq); } -static int32_t mndProcessConfigReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - SConfigReq configReq = {0}; - SDnodeObj *pDnode = NULL; - int32_t code = -1; - tDeserializeSConfigReq(pReq->pCont, pReq->contLen, &configReq); - SArray *diffArray = taosArrayInit(16, sizeof(SConfigItem)); - SConfigRsp configRsp = {0}; - configRsp.forceReadConfig = configReq.forceReadConfig; - configRsp.cver = tsmmConfigVersion; - if (configRsp.forceReadConfig) { - // compare config array from configReq with current config array - if (compareSConfigItemArrays(getGlobalCfg(tsCfg), configReq.array, diffArray)) { - configRsp.array = diffArray; - } else { - configRsp.isConifgVerified = 1; - taosArrayDestroy(diffArray); - } - } else { - configRsp.array = getGlobalCfg(tsCfg); - if (configReq.cver == tsmmConfigVersion) { - configRsp.isVersionVerified = 1; - } else { - configRsp.array = getGlobalCfg(tsCfg); - } - } - - int32_t contLen = tSerializeSConfigRsp(NULL, 0, &configRsp); - void *pHead = rpcMallocCont(contLen); - contLen = tSerializeSConfigRsp(pHead, contLen, &configRsp); - taosArrayDestroy(diffArray); - if (contLen < 0) { - code = contLen; - goto _OVER; - } - pReq->info.rspLen = contLen; - pReq->info.rsp = pHead; -_OVER: - - mndReleaseDnode(pMnode, pDnode); - return mndUpdClusterInfo(pReq); -} static int32_t mndProcessNotifyReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 6c30193ea7..4cdad9825f 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -15,11 +15,12 @@ #define _DEFAULT_SOURCE #include "mndAcct.h" -#include "mndArbGroup.h" #include "mndAnode.h" +#include "mndArbGroup.h" #include "mndCluster.h" #include "mndCompact.h" #include "mndCompactDetail.h" +#include "mndConfig.h" #include "mndConsumer.h" #include "mndDb.h" #include "mndDnode.h" @@ -603,6 +604,7 @@ static int32_t mndAllocStep(SMnode *pMnode, char *name, MndInitFp initFp, MndCle static int32_t mndInitSteps(SMnode *pMnode) { TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-wal", mndInitWal, mndCloseWal)); + TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-config", mndInitConfig, mndCleanupArbGroup)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-sdb", mndInitSdb, mndCleanupSdb)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-trans", mndInitTrans, mndCleanupTrans)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-cluster", mndInitCluster, mndCleanupCluster)); diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index f6d1587bb2..c9b352d79e 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -60,6 +60,7 @@ extern "C" { #define SDB_GET_INT16(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawInt16, int16_t) #define SDB_GET_INT8(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawInt8, int8_t) #define SDB_GET_UINT8(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawUInt8, uint8_t) +#define SDB_GET_BOOL(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawBool, bool) #define SDB_GET_RESERVE(pRaw, dataPos, valLen, pos) \ { \ @@ -81,6 +82,7 @@ extern "C" { #define SDB_SET_INT16(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawInt16, int16_t) #define SDB_SET_INT8(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawInt8, int8_t) #define SDB_SET_UINT8(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawUInt8, uint8_t) +#define SDB_SET_BOOL(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawBool, bool) #define SDB_SET_BINARY(pRaw, dataPos, val, valLen, pos) \ { \ @@ -162,7 +164,8 @@ typedef enum { SDB_GRANT = 26, // grant log SDB_ARBGROUP = 27, SDB_ANODE = 28, - SDB_MAX = 29 + SDB_MAX = 29, + SDB_CFG = 30 } ESdbType; typedef struct SSdbRaw { @@ -373,7 +376,7 @@ int32_t sdbGetSize(SSdb *pSdb, ESdbType type); /** * @brief get valid number of rows, removed rows are ignored */ -int32_t sdbGetValidSize(SSdb* pSdb, ESdbType type); +int32_t sdbGetValidSize(SSdb *pSdb, ESdbType type); /** * @brief Get the max id of the table, keyType of table should be INT32 @@ -407,6 +410,7 @@ SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen); void sdbFreeRaw(SSdbRaw *pRaw); int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val); int32_t sdbSetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t val); +int32_t sdbSetRawBool(SSdbRaw *pRaw, int32_t dataPos, bool val); int32_t sdbSetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t val); int32_t sdbSetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t val); int32_t sdbSetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t val); @@ -415,6 +419,7 @@ int32_t sdbSetRawDataLen(SSdbRaw *pRaw, int32_t dataLen); int32_t sdbSetRawStatus(SSdbRaw *pRaw, ESdbStatus status); int32_t sdbGetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t *val); int32_t sdbGetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t *val); +int32_t sdbGetRawBool(SSdbRaw *pRaw, int32_t dataPos, bool *val); int32_t sdbGetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t *val); int32_t sdbGetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t *val); int32_t sdbGetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t *val); diff --git a/source/dnode/mnode/sdb/src/sdbRaw.c b/source/dnode/mnode/sdb/src/sdbRaw.c index 24ac2ac462..c71e415d1b 100644 --- a/source/dnode/mnode/sdb/src/sdbRaw.c +++ b/source/dnode/mnode/sdb/src/sdbRaw.c @@ -84,6 +84,14 @@ int32_t sdbSetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t val) { return 0; } +int32_t sdbSetRawBool(SSdbRaw *pRaw, int32_t dataPos, bool val) { + if (val) { + return sdbSetRawUInt8(pRaw, dataPos, 1); + } else { + return sdbSetRawUInt8(pRaw, dataPos, 0); + } +} + int32_t sdbSetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t val) { int32_t code = 0; if (pRaw == NULL) { @@ -214,6 +222,21 @@ int32_t sdbGetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t *val) { return 0; } +int32_t sdbGetRawBool(SSdbRaw *pRaw, int32_t dataPos, bool *val) { + int32_t code = 0; + uint8_t v = 0; + code = sdbGetRawUInt8(pRaw, dataPos, &v); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (v) { + *val = true; + } else { + *val = false; + } + return TSDB_CODE_SUCCESS; +} + int32_t sdbGetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t *val) { int32_t code = 0; if (pRaw == NULL) { diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 3fed23e8a2..ca3b123472 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -404,6 +404,47 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy TAOS_RETURN(code); } +int32_t cfgUpdateItem(SConfigItem *pItem, SConfigItem *newItem) { + int32_t code = TSDB_CODE_SUCCESS; + if (pItem == NULL || newItem == NULL) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } + + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: { + pItem->bval = newItem->bval; + break; + } + case CFG_DTYPE_INT32: { + pItem->i32 = newItem->i32; + break; + } + case CFG_DTYPE_INT64: { + pItem->i64 = newItem->i64; + break; + } + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: { + pItem->fval = newItem->fval; + break; + } + case CFG_DTYPE_DIR: + case CFG_DTYPE_TIMEZONE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_NONE: + case CFG_DTYPE_STRING: { + pItem->str = newItem->str; + break; + } + default: + code = TSDB_CODE_INVALID_CFG; + break; + } + + TAOS_RETURN(code); +} + SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName) { if (pCfg == NULL) return NULL; int32_t size = taosArrayGetSize(pCfg->localArray); From 4a03483683e7205f3acb4fbed38d4cde3286fbe2 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Fri, 15 Nov 2024 14:39:10 +0800 Subject: [PATCH 19/99] add prepare func. --- source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 5 --- source/dnode/mnode/impl/inc/mndConfig.h | 5 ++- source/dnode/mnode/impl/src/mndConfig.c | 57 +++++++++++------------- source/dnode/mnode/impl/src/mndMain.c | 9 +++- source/dnode/mnode/sdb/inc/sdb.h | 15 ++++++- source/dnode/mnode/sdb/src/sdbFile.c | 29 ++++++++++++ source/dnode/mnode/sdb/src/sdbHash.c | 8 ++-- 7 files changed, 85 insertions(+), 43 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index b3b740f0fb..b1b7a90db8 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -45,10 +45,6 @@ static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) { return code; } -static void mmBuildConfigForDeploy(SMnodeMgmt *pMgmt) { - persistGlobalConfig(getGlobalCfg(tsCfg), pMgmt->path, tsmmConfigVersion); -} - static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInput, SMnodeOpt *pOption) { pOption->deploy = true; pOption->msgCb = pMgmt->msgCb; @@ -124,7 +120,6 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { dInfo("mnode start to deploy"); pMgmt->pData->dnodeId = 1; mmBuildOptionForDeploy(pMgmt, pInput, &option); - mmBuildConfigForDeploy(pMgmt); } else { dInfo("mnode start to open"); mmBuildOptionForOpen(pMgmt, &option); diff --git a/source/dnode/mnode/impl/inc/mndConfig.h b/source/dnode/mnode/impl/inc/mndConfig.h index 22a115b30b..b054e3c855 100644 --- a/source/dnode/mnode/impl/inc/mndConfig.h +++ b/source/dnode/mnode/impl/inc/mndConfig.h @@ -21,12 +21,15 @@ #ifdef __cplusplus extern "C" { #endif -int32_t mndInitConfig(SMnode *pMnode); +int32_t mndInitConfig(SMnode *pMnode); + SSdbRaw *mnCfgActionEncode(SConfigItem *pCfg); SSdbRow *mndCfgActionDecode(SSdbRaw *pRaw); static int32_t mndCfgActionInsert(SSdb *pSdb, SConfigItem *item); static int32_t mndCfgActionDelete(SSdb *pSdb, SConfigItem *item); static int32_t mndCfgActionUpdate(SSdb *pSdb, SConfigItem *oldItem, SConfigItem *newItem); +static int32_t mndCfgActionDeploy(SMnode *pMnode); +static int32_t mndCfgActionPrepare(SMnode *pMnode); static int32_t mndProcessConfigReq(SRpcMsg *pReq); #ifdef __cplusplus diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index 59fdc6260a..bf864d64b8 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -29,28 +29,22 @@ int32_t mndSetCreateConfigCommitLogs(STrans *pTrans, SConfigItem *item); int32_t mndInitConfig(SMnode *pMnode) { int32_t code = 0; - SSdbTable table = { - .sdbType = SDB_CFG, - .keyType = SDB_KEY_INT32, - .encodeFp = (SdbEncodeFp)mnCfgActionEncode, - .decodeFp = (SdbDecodeFp)mndCfgActionDecode, - .insertFp = (SdbInsertFp)mndCfgActionInsert, - .updateFp = (SdbUpdateFp)mndCfgActionUpdate, - .deleteFp = (SdbDeleteFp)mndCfgActionDelete, - }; - - if (pMnode->deploy) { - mndInitWriteCfg(pMnode); - } else { - mndInitReadCfg(pMnode); - } + SSdbTable table = {.sdbType = SDB_CFG, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mnCfgActionEncode, + .decodeFp = (SdbDecodeFp)mndCfgActionDecode, + .insertFp = (SdbInsertFp)mndCfgActionInsert, + .updateFp = (SdbUpdateFp)mndCfgActionUpdate, + .deleteFp = (SdbDeleteFp)mndCfgActionDelete, + .deployFp = (SdbDeployFp)mndCfgActionDeploy, + .prepareFp = (SdbPrepareFp)mndCfgActionPrepare}; mndSetMsgHandle(pMnode, TDMT_MND_CONFIG, mndProcessConfigReq); return sdbSetTable(pMnode->pSdb, table); } -SSdbRaw *mnCfgActionEncode(SConfigItem *pCfg) { +SSdbRaw *mnCfgActionEncode(SConfigItem *pItem) { int32_t code = 0; int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -60,26 +54,25 @@ SSdbRaw *mnCfgActionEncode(SConfigItem *pCfg) { SSdbRaw *pRaw = sdbAllocRaw(SDB_CFG, CFG_VER_NUMBER, size); if (pRaw == NULL) goto _OVER; - int32_t dataPos = 0; - SConfigItem *item = NULL; - SDB_SET_INT32(pRaw, dataPos, strlen(item->name), _OVER) - SDB_SET_BINARY(pRaw, dataPos, item->name, strlen(item->name), _OVER) - SDB_SET_INT32(pRaw, dataPos, item->dtype, _OVER) - switch (item->dtype) { + int32_t dataPos = 0; + SDB_SET_INT32(pRaw, dataPos, strlen(pItem->name), _OVER) + SDB_SET_BINARY(pRaw, dataPos, pItem->name, strlen(pItem->name), _OVER) + SDB_SET_INT32(pRaw, dataPos, pItem->dtype, _OVER) + switch (pItem->dtype) { case CFG_DTYPE_NONE: break; case CFG_DTYPE_BOOL: - SDB_SET_BOOL(pRaw, dataPos, item->bval, _OVER) + SDB_SET_BOOL(pRaw, dataPos, pItem->bval, _OVER) break; case CFG_DTYPE_INT32: - SDB_SET_INT32(pRaw, dataPos, item->i32, _OVER); + SDB_SET_INT32(pRaw, dataPos, pItem->i32, _OVER); break; case CFG_DTYPE_INT64: - SDB_SET_INT64(pRaw, dataPos, item->i64, _OVER); + SDB_SET_INT64(pRaw, dataPos, pItem->i64, _OVER); break; case CFG_DTYPE_FLOAT: case CFG_DTYPE_DOUBLE: - (void)sprintf(buf, "%f", item->fval); + (void)sprintf(buf, "%f", pItem->fval); SDB_SET_INT32(pRaw, dataPos, strlen(buf), _OVER) SDB_SET_BINARY(pRaw, dataPos, buf, strlen(buf), _OVER) break; @@ -88,8 +81,8 @@ SSdbRaw *mnCfgActionEncode(SConfigItem *pCfg) { case CFG_DTYPE_LOCALE: case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: - SDB_SET_INT32(pRaw, dataPos, strlen(item->str), _OVER) - SDB_SET_BINARY(pRaw, dataPos, item->str, strlen(item->str), _OVER) + SDB_SET_INT32(pRaw, dataPos, strlen(pItem->str), _OVER) + SDB_SET_BINARY(pRaw, dataPos, pItem->str, strlen(pItem->str), _OVER) break; } @@ -101,7 +94,7 @@ _OVER: sdbFreeRaw(pRaw); return NULL; } - mTrace("cfg encode to raw:%p, row:%p", pRaw, pCfg); + mTrace("cfg encode to raw:%p, row:%p", pRaw, pItem); return pRaw; } @@ -121,7 +114,7 @@ SSdbRow *mndCfgActionDecode(SSdbRaw *pRaw) { goto _OVER; } - pRow = sdbAllocRow(sizeof(SSdbRaw)); + pRow = sdbAllocRow(sizeof(SConfigItem)); if (pRow == NULL) goto _OVER; item = sdbGetRowObj(pRow); @@ -184,6 +177,10 @@ static int32_t mndCfgActionUpdate(SSdb *pSdb, SConfigItem *pOld, SConfigItem *pN return 0; } +static int32_t mndCfgActionDeploy(SMnode *pMnode) { return mndInitWriteCfg(pMnode); } + +static int32_t mndCfgActionPrepare(SMnode *pMnode) { return mndInitConfig(pMnode); } + static int32_t mndProcessConfigReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SConfigReq configReq = {0}; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 4cdad9825f..8fdbefa496 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -604,7 +604,6 @@ static int32_t mndAllocStep(SMnode *pMnode, char *name, MndInitFp initFp, MndCle static int32_t mndInitSteps(SMnode *pMnode) { TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-wal", mndInitWal, mndCloseWal)); - TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-config", mndInitConfig, mndCleanupArbGroup)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-sdb", mndInitSdb, mndCleanupSdb)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-trans", mndInitTrans, mndCleanupTrans)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-cluster", mndInitCluster, mndCleanupCluster)); @@ -639,7 +638,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-query", mndInitQuery, mndCleanupQuery)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-sync", mndInitSync, mndCleanupSync)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-telem", mndInitTelem, mndCleanupTelem)); - + TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-config", mndInitConfig, mndCleanupArbGroup)); return 0; } @@ -798,6 +797,12 @@ int32_t mndStart(SMnode *pMnode) { return -1; } mndSetRestored(pMnode, true); + } else { + if (sdbPrepare(pMnode->pSdb) != 0) { + mError("failed to prepare sdb while start mnode"); + return -1; + } + mndSetRestored(pMnode, true); } grantReset(pMnode, TSDB_GRANT_ALL, 0); diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index c9b352d79e..19a9343cea 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -115,6 +115,7 @@ typedef int32_t (*SdbInsertFp)(SSdb *pSdb, void *pObj); typedef int32_t (*SdbUpdateFp)(SSdb *pSdb, void *pSrcObj, void *pDstObj); typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj, bool callFunc); typedef int32_t (*SdbDeployFp)(SMnode *pMnode); +typedef int32_t (*SdbPrepareFp)(SMnode *pMnode); typedef int32_t (*SdbValidateFp)(SMnode *pMnode, void *pTrans, SSdbRaw *pRaw); typedef SSdbRow *(*SdbDecodeFp)(SSdbRaw *pRaw); typedef SSdbRaw *(*SdbEncodeFp)(void *pObj); @@ -164,8 +165,8 @@ typedef enum { SDB_GRANT = 26, // grant log SDB_ARBGROUP = 27, SDB_ANODE = 28, - SDB_MAX = 29, - SDB_CFG = 30 + SDB_CFG = 29, + SDB_MAX = 30 } ESdbType; typedef struct SSdbRaw { @@ -205,6 +206,7 @@ typedef struct SSdb { SdbUpdateFp updateFps[SDB_MAX]; SdbDeleteFp deleteFps[SDB_MAX]; SdbDeployFp deployFps[SDB_MAX]; + SdbPrepareFp prepareFps[SDB_MAX]; SdbEncodeFp encodeFps[SDB_MAX]; SdbDecodeFp decodeFps[SDB_MAX]; SdbValidateFp validateFps[SDB_MAX]; @@ -221,6 +223,7 @@ typedef struct { ESdbType sdbType; EKeyType keyType; SdbDeployFp deployFp; + SdbPrepareFp prepareFp; SdbEncodeFp encodeFp; SdbDecodeFp decodeFp; SdbInsertFp insertFp; @@ -268,6 +271,14 @@ int32_t sdbSetTable(SSdb *pSdb, SSdbTable table); */ int32_t sdbDeploy(SSdb *pSdb); +/** + * @brief prepare the initial rows of sdb. + * + * @param pSdb The sdb object. + * @return int32_t 0 for success, -1 for failure. + */ +int32_t sdbPrepare(SSdb *pSdb); + /** * @brief Load sdb from file. * diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 474b22cca0..64cdcbee9e 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -48,6 +48,26 @@ static int32_t sdbDeployData(SSdb *pSdb) { return 0; } +static int32_t sdbPrepareData(SSdb *pSdb) { + int32_t code = 0; + mInfo("start to prepare sdb"); + + for (int32_t i = SDB_MAX - 1; i >= 0; --i) { + SdbDeployFp fp = pSdb->prepareFps[i]; + if (fp == NULL) continue; + + mInfo("start to prepare sdb:%s", sdbTableName(i)); + code = (*fp)(pSdb->pMnode); + if (code != 0) { + mError("failed to prepare sdb:%s since %s", sdbTableName(i), tstrerror(code)); + return -1; + } + } + + mInfo("sdb prepare success"); + return 0; +} + static void sdbResetData(SSdb *pSdb) { mInfo("start to reset sdb"); @@ -641,6 +661,15 @@ int32_t sdbDeploy(SSdb *pSdb) { return 0; } +int32_t sdbPrepare(SSdb *pSdb) { + int32_t code = 0; + code = sdbPrepareData(pSdb); + if (code != 0) { + TAOS_RETURN(code); + } + return 0; +} + static SSdbIter *sdbCreateIter(SSdb *pSdb) { SSdbIter *pIter = taosMemoryCalloc(1, sizeof(SSdbIter)); if (pIter == NULL) { diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 3f85ccb087..b159190119 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -76,6 +76,8 @@ const char *sdbTableName(ESdbType type) { return "arb_group"; case SDB_ANODE: return "anode"; + case SDB_CFG: + return "config"; default: return "undefine"; } @@ -444,7 +446,7 @@ void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStat void sdbCancelFetch(SSdb *pSdb, void *pIter) { if (pIter == NULL) return; - SSdbRow *pRow = *(SSdbRow **)pIter; + SSdbRow *pRow = *(SSdbRow **)pIter; mTrace("cancel fetch row:%p", pRow); SHashObj *hash = sdbGetHash(pSdb, pRow->type); if (hash == NULL) return; @@ -532,12 +534,12 @@ int64_t sdbGetTableVer(SSdb *pSdb, ESdbType type) { } bool countValid(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { - int32_t* pInt = p1; + int32_t *pInt = p1; (*pInt) += 1; return true; } -int32_t sdbGetValidSize(SSdb* pSdb, ESdbType type) { +int32_t sdbGetValidSize(SSdb *pSdb, ESdbType type) { int32_t num = 0; sdbTraverse(pSdb, type, countValid, &num, 0, 0); return num; From 9b9b3e8e96bfbeee267efe22e0e86a9372148a3a Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 18 Nov 2024 12:05:08 +0800 Subject: [PATCH 20/99] Write configs to mnd sdb. --- source/dnode/mnode/impl/src/mndConfig.c | 7 +++++-- source/dnode/mnode/impl/src/mndMain.c | 2 +- source/dnode/mnode/sdb/src/sdb.c | 1 + 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index bf864d64b8..74cd4a2819 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -121,7 +121,9 @@ SSdbRow *mndCfgActionDecode(SSdbRaw *pRaw) { if (item == NULL) goto _OVER; int32_t dataPos = 0; SDB_GET_INT32(pRaw, dataPos, &len, _OVER) - SDB_GET_BINARY(pRaw, dataPos, item->name, len, _OVER) + char *buf = taosMemoryMalloc(len + 1); + SDB_GET_BINARY(pRaw, dataPos, buf, len, _OVER) + item->name = buf; SDB_GET_INT32(pRaw, dataPos, (int32_t *)&item->dtype, _OVER) switch (item->dtype) { case CFG_DTYPE_NONE: @@ -150,6 +152,7 @@ SSdbRow *mndCfgActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, item->str, len, _OVER) break; } + terrno = TSDB_CODE_SUCCESS; _OVER: if (terrno != 0) { @@ -179,7 +182,7 @@ static int32_t mndCfgActionUpdate(SSdb *pSdb, SConfigItem *pOld, SConfigItem *pN static int32_t mndCfgActionDeploy(SMnode *pMnode) { return mndInitWriteCfg(pMnode); } -static int32_t mndCfgActionPrepare(SMnode *pMnode) { return mndInitConfig(pMnode); } +static int32_t mndCfgActionPrepare(SMnode *pMnode) { return mndInitReadCfg(pMnode); } static int32_t mndProcessConfigReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 8fdbefa496..39de235920 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -638,7 +638,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-query", mndInitQuery, mndCleanupQuery)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-sync", mndInitSync, mndCleanupSync)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-telem", mndInitTelem, mndCleanupTelem)); - TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-config", mndInitConfig, mndCleanupArbGroup)); + TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-config", mndInitConfig, NULL)); return 0; } diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index 8d0898e8ac..6a273f9ab3 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -127,6 +127,7 @@ int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) { pSdb->deployFps[sdbType] = table.deployFp; pSdb->encodeFps[sdbType] = table.encodeFp; pSdb->decodeFps[sdbType] = table.decodeFp; + pSdb->prepareFps[sdbType] = table.prepareFp; pSdb->validateFps[sdbType] = table.validateFp; int32_t hashType = 0; From 4b8feab900dc4b837cda33ba6519a92e1b45801c Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 18 Nov 2024 19:52:41 +0800 Subject: [PATCH 21/99] fix mnd config persist. --- include/util/tconfig.h | 1 - source/dnode/mnode/impl/inc/mndConfig.h | 8 +- source/dnode/mnode/impl/inc/mndDef.h | 21 ++++ source/dnode/mnode/impl/src/mndConfig.c | 142 ++++++++++++++++-------- source/dnode/mnode/impl/src/mndDef.c | 79 +++++++++---- source/dnode/mnode/impl/src/mndMain.c | 2 +- source/dnode/mnode/sdb/src/sdbFile.c | 2 +- source/dnode/mnode/sdb/src/sdbHash.c | 3 + source/util/src/tconfig.c | 40 ------- 9 files changed, 179 insertions(+), 119 deletions(-) diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 312a10e2d6..98e0df9655 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -118,7 +118,6 @@ void cfgCleanup(SConfig *pCfg); int32_t cfgGetSize(SConfig *pCfg); SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName); int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype, bool lock); -int32_t cfgUpdateItem(SConfigItem *pItem, SConfigItem *newItem); int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer); int32_t cfgCreateIter(SConfig *pConf, SConfigIter **ppIter); diff --git a/source/dnode/mnode/impl/inc/mndConfig.h b/source/dnode/mnode/impl/inc/mndConfig.h index b054e3c855..bbfa3f4a65 100644 --- a/source/dnode/mnode/impl/inc/mndConfig.h +++ b/source/dnode/mnode/impl/inc/mndConfig.h @@ -23,11 +23,11 @@ extern "C" { #endif int32_t mndInitConfig(SMnode *pMnode); -SSdbRaw *mnCfgActionEncode(SConfigItem *pCfg); +SSdbRaw *mnCfgActionEncode(SConfigObj *pCfg); SSdbRow *mndCfgActionDecode(SSdbRaw *pRaw); -static int32_t mndCfgActionInsert(SSdb *pSdb, SConfigItem *item); -static int32_t mndCfgActionDelete(SSdb *pSdb, SConfigItem *item); -static int32_t mndCfgActionUpdate(SSdb *pSdb, SConfigItem *oldItem, SConfigItem *newItem); +static int32_t mndCfgActionInsert(SSdb *pSdb, SConfigObj *obj); +static int32_t mndCfgActionDelete(SSdb *pSdb, SConfigObj *obj); +static int32_t mndCfgActionUpdate(SSdb *pSdb, SConfigObj *oldItem, SConfigObj *newObj); static int32_t mndCfgActionDeploy(SMnode *pMnode); static int32_t mndCfgActionPrepare(SMnode *pMnode); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index c2ee3944e5..8770704211 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -316,6 +316,27 @@ typedef struct { TdThreadMutex mutex; } SArbGroup; +typedef struct { + char name[CFG_NAME_MAX_LEN]; + ECfgDataType dtype; + union { + bool bval; + float fval; + int32_t i32; + int64_t i64; + char* str; + }; + union { + int64_t imin; + float fmin; + }; + union { + int64_t imax; + float fmax; + }; +} SConfigObj; + +SConfigObj* mndInitConfigObj(SConfigItem* pItem); typedef struct { int32_t maxUsers; diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index 74cd4a2819..fbe44759fd 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -21,11 +21,12 @@ #define CFG_VER_NUMBER 1 #define CFG_RESERVE_SIZE 63 +static int32_t cfgUpdateItem(SConfigItem *pItem, SConfigObj *obj); static int32_t mndProcessConfigReq(SRpcMsg *pReq); static int32_t mndInitWriteCfg(SMnode *pMnode); static int32_t mndInitReadCfg(SMnode *pMnode); -int32_t mndSetCreateConfigCommitLogs(STrans *pTrans, SConfigItem *item); +int32_t mndSetCreateConfigCommitLogs(STrans *pTrans, SConfigObj *obj); int32_t mndInitConfig(SMnode *pMnode) { int32_t code = 0; @@ -44,35 +45,36 @@ int32_t mndInitConfig(SMnode *pMnode) { return sdbSetTable(pMnode->pSdb, table); } -SSdbRaw *mnCfgActionEncode(SConfigItem *pItem) { +SSdbRaw *mnCfgActionEncode(SConfigObj *obj) { int32_t code = 0; int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; char buf[30]; - int32_t size = sizeof(SConfigItem) + CFG_RESERVE_SIZE; + int32_t size = sizeof(SConfigObj); SSdbRaw *pRaw = sdbAllocRaw(SDB_CFG, CFG_VER_NUMBER, size); if (pRaw == NULL) goto _OVER; int32_t dataPos = 0; - SDB_SET_INT32(pRaw, dataPos, strlen(pItem->name), _OVER) - SDB_SET_BINARY(pRaw, dataPos, pItem->name, strlen(pItem->name), _OVER) - SDB_SET_INT32(pRaw, dataPos, pItem->dtype, _OVER) - switch (pItem->dtype) { + char name[CFG_NAME_MAX_LEN] = {0}; + strncpy(name, obj->name, CFG_NAME_MAX_LEN); + SDB_SET_BINARY(pRaw, dataPos, name, CFG_NAME_MAX_LEN, _OVER) + SDB_SET_INT32(pRaw, dataPos, obj->dtype, _OVER) + switch (obj->dtype) { case CFG_DTYPE_NONE: break; case CFG_DTYPE_BOOL: - SDB_SET_BOOL(pRaw, dataPos, pItem->bval, _OVER) + SDB_SET_BOOL(pRaw, dataPos, obj->bval, _OVER) break; case CFG_DTYPE_INT32: - SDB_SET_INT32(pRaw, dataPos, pItem->i32, _OVER); + SDB_SET_INT32(pRaw, dataPos, obj->i32, _OVER); break; case CFG_DTYPE_INT64: - SDB_SET_INT64(pRaw, dataPos, pItem->i64, _OVER); + SDB_SET_INT64(pRaw, dataPos, obj->i64, _OVER); break; case CFG_DTYPE_FLOAT: case CFG_DTYPE_DOUBLE: - (void)sprintf(buf, "%f", pItem->fval); + (void)sprintf(buf, "%f", obj->fval); SDB_SET_INT32(pRaw, dataPos, strlen(buf), _OVER) SDB_SET_BINARY(pRaw, dataPos, buf, strlen(buf), _OVER) break; @@ -81,8 +83,8 @@ SSdbRaw *mnCfgActionEncode(SConfigItem *pItem) { case CFG_DTYPE_LOCALE: case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: - SDB_SET_INT32(pRaw, dataPos, strlen(pItem->str), _OVER) - SDB_SET_BINARY(pRaw, dataPos, pItem->str, strlen(pItem->str), _OVER) + SDB_SET_INT32(pRaw, dataPos, strlen(obj->str), _OVER) + SDB_SET_BINARY(pRaw, dataPos, obj->str, strlen(obj->str), _OVER) break; } @@ -94,7 +96,7 @@ _OVER: sdbFreeRaw(pRaw); return NULL; } - mTrace("cfg encode to raw:%p, row:%p", pRaw, pItem); + mTrace("cfg encode to raw:%p, row:%p", pRaw, obj); return pRaw; } @@ -103,8 +105,8 @@ SSdbRow *mndCfgActionDecode(SSdbRaw *pRaw) { int32_t lino = 0; int32_t len = -1; terrno = TSDB_CODE_OUT_OF_MEMORY; - SSdbRow *pRow = NULL; - SConfigItem *item = NULL; + SSdbRow *pRow = NULL; + SConfigObj *obj = NULL; int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; @@ -114,28 +116,26 @@ SSdbRow *mndCfgActionDecode(SSdbRaw *pRaw) { goto _OVER; } - pRow = sdbAllocRow(sizeof(SConfigItem)); + pRow = sdbAllocRow(sizeof(SConfigObj)); if (pRow == NULL) goto _OVER; - item = sdbGetRowObj(pRow); - if (item == NULL) goto _OVER; + obj = sdbGetRowObj(pRow); + if (obj == NULL) goto _OVER; int32_t dataPos = 0; - SDB_GET_INT32(pRaw, dataPos, &len, _OVER) - char *buf = taosMemoryMalloc(len + 1); - SDB_GET_BINARY(pRaw, dataPos, buf, len, _OVER) - item->name = buf; - SDB_GET_INT32(pRaw, dataPos, (int32_t *)&item->dtype, _OVER) - switch (item->dtype) { + // TODO(beryl):free it. + SDB_GET_BINARY(pRaw, dataPos, obj->name, CFG_NAME_MAX_LEN, _OVER) + SDB_GET_INT32(pRaw, dataPos, (int32_t *)&obj->dtype, _OVER) + switch (obj->dtype) { case CFG_DTYPE_NONE: break; case CFG_DTYPE_BOOL: - SDB_GET_BOOL(pRaw, dataPos, &item->bval, _OVER) + SDB_GET_BOOL(pRaw, dataPos, &obj->bval, _OVER) break; case CFG_DTYPE_INT32: - SDB_GET_INT32(pRaw, dataPos, &item->i32, _OVER); + SDB_GET_INT32(pRaw, dataPos, &obj->i32, _OVER); break; case CFG_DTYPE_INT64: - SDB_GET_INT64(pRaw, dataPos, &item->i64, _OVER); + SDB_GET_INT64(pRaw, dataPos, &obj->i64, _OVER); break; case CFG_DTYPE_FLOAT: case CFG_DTYPE_DOUBLE: @@ -149,7 +149,7 @@ SSdbRow *mndCfgActionDecode(SSdbRaw *pRaw) { case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: SDB_GET_INT32(pRaw, dataPos, &len, _OVER) - SDB_GET_BINARY(pRaw, dataPos, item->str, len, _OVER) + SDB_GET_BINARY(pRaw, dataPos, obj->str, len, _OVER) break; } terrno = TSDB_CODE_SUCCESS; @@ -161,21 +161,21 @@ _OVER: return NULL; } - mTrace("cfg decode from raw:%p, row:%p", pRaw, item); + mTrace("cfg decode from raw:%p, row:%p", pRaw, obj); return pRow; } -static int32_t mndCfgActionInsert(SSdb *pSdb, SConfigItem *item) { - mTrace("cfg:%s, perform insert action, row:%p", item->name, item); +static int32_t mndCfgActionInsert(SSdb *pSdb, SConfigObj *obj) { + mTrace("cfg:%s, perform insert action, row:%p", obj->name, obj); return 0; } -static int32_t mndCfgActionDelete(SSdb *pSdb, SConfigItem *item) { - mTrace("cfg:%s, perform delete action, row:%p", item->name, item); +static int32_t mndCfgActionDelete(SSdb *pSdb, SConfigObj *obj) { + mTrace("cfg:%s, perform delete action, row:%p", obj->name, obj); return 0; } -static int32_t mndCfgActionUpdate(SSdb *pSdb, SConfigItem *pOld, SConfigItem *pNew) { +static int32_t mndCfgActionUpdate(SSdb *pSdb, SConfigObj *pOld, SConfigObj *pNew) { mTrace("cfg:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); return 0; } @@ -232,25 +232,27 @@ int32_t mndInitWriteCfg(SMnode *pMnode) { int code = -1; size_t sz = 0; - SConfigItem item = {0}; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "init-write-config"); + SConfigObj obj = {0}; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "init-write-config"); if (pTrans == NULL) { mError("failed to init write cfg in create trans, since %s", terrstr()); goto _OVER; } // encode mnd config version - item = (SConfigItem){.name = "tsmmConfigVersion", .dtype = CFG_DTYPE_INT32, .i32 = tsmmConfigVersion}; - if ((code = mndSetCreateConfigCommitLogs(pTrans, &item)) != 0) { + obj = (SConfigObj){.name = "tsmmConfigVersion", .dtype = CFG_DTYPE_INT32, .i32 = tsmmConfigVersion}; + if ((code = mndSetCreateConfigCommitLogs(pTrans, &obj)) != 0) { mError("failed to init mnd config version, since %s", terrstr()); } sz = taosArrayGetSize(getGlobalCfg(tsCfg)); for (int i = 0; i < sz; ++i) { SConfigItem *item = taosArrayGet(getGlobalCfg(tsCfg), i); - if ((code = mndSetCreateConfigCommitLogs(pTrans, item)) != 0) { + SConfigObj *obj = mndInitConfigObj(item); + if ((code = mndSetCreateConfigCommitLogs(pTrans, obj)) != 0) { mError("failed to init mnd config:%s, since %s", item->name, terrstr()); } + taosMemoryFree(obj); } if ((code = mndTransCheckConflict(pMnode, pTrans)) != 0) goto _OVER; if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER; @@ -268,31 +270,32 @@ int32_t mndInitReadCfg(SMnode *pMnode) { mError("failed to init read cfg in create trans, since %s", terrstr()); goto _OVER; } - SConfigItem *item = sdbAcquire(pMnode->pSdb, SDB_CFG, "tsmmConfigVersion"); - if (item == NULL) { + SConfigObj *obj = sdbAcquire(pMnode->pSdb, SDB_CFG, "tsmmConfigVersion"); + if (obj == NULL) { mInfo("failed to acquire mnd config version, since %s", terrstr()); goto _OVER; } else { - tsmmConfigVersion = item->i32; - sdbRelease(pMnode->pSdb, item); + tsmmConfigVersion = obj->i32; + sdbRelease(pMnode->pSdb, obj); } + sz = taosArrayGetSize(getGlobalCfg(tsCfg)); for (int i = 0; i < sz; ++i) { SConfigItem *item = taosArrayGet(getGlobalCfg(tsCfg), i); - SConfigItem *newItem = sdbAcquire(pMnode->pSdb, SDB_CFG, item->name); - if (newItem == NULL) { + SConfigObj *newObj = sdbAcquire(pMnode->pSdb, SDB_CFG, item->name); + if (newObj == NULL) { mInfo("failed to acquire mnd config:%s, since %s", item->name, terrstr()); continue; } - cfgUpdateItem(item, newItem); - sdbRelease(pMnode->pSdb, newItem); + cfgUpdateItem(item, newObj); + sdbRelease(pMnode->pSdb, newObj); } _OVER: mndTransDrop(pTrans); return TSDB_CODE_SUCCESS; } -int32_t mndSetCreateConfigCommitLogs(STrans *pTrans, SConfigItem *item) { +int32_t mndSetCreateConfigCommitLogs(STrans *pTrans, SConfigObj *item) { int32_t code = 0; SSdbRaw *pCommitRaw = mnCfgActionEncode(item); if (pCommitRaw == NULL) { @@ -302,4 +305,45 @@ int32_t mndSetCreateConfigCommitLogs(STrans *pTrans, SConfigItem *item) { if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw) != 0)) TAOS_RETURN(code); if ((code = sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)) != 0) TAOS_RETURN(code); return TSDB_CODE_SUCCESS; +} + +int32_t cfgUpdateItem(SConfigItem *pItem, SConfigObj *obj) { + int32_t code = TSDB_CODE_SUCCESS; + if (pItem == NULL || obj == NULL) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } + + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: { + pItem->bval = obj->bval; + break; + } + case CFG_DTYPE_INT32: { + pItem->i32 = obj->i32; + break; + } + case CFG_DTYPE_INT64: { + pItem->i64 = obj->i64; + break; + } + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: { + pItem->fval = obj->fval; + break; + } + case CFG_DTYPE_DIR: + case CFG_DTYPE_TIMEZONE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_NONE: + case CFG_DTYPE_STRING: { + pItem->str = obj->str; + break; + } + default: + code = TSDB_CODE_INVALID_CFG; + break; + } + + TAOS_RETURN(code); } \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 61606b0364..ad2c3f3b60 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ #define _DEFAULT_SOURCE -#include "mndDef.h" #include "mndConsumer.h" +#include "mndDef.h" #include "taoserror.h" static void *freeStreamTasks(SArray *pTaskLevel); @@ -72,7 +72,7 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { TAOS_CHECK_RETURN(tEncodeI32(pEncoder, innerSz)); for (int32_t j = 0; j < innerSz; j++) { SStreamTask *pTask = taosArrayGetP(pArray, j); - if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER){ + if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER) { pTask->ver = SSTREAM_TASK_VER; } TAOS_CHECK_RETURN(tEncodeStreamTask(pEncoder, pTask)); @@ -273,9 +273,9 @@ void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp, int8_t sver) { static void *topicNameDup(void *p) { return taosStrdup((char *)p); } -int32_t tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType, - char *topic, SCMSubscribeReq *subscribe, SMqConsumerObj** ppConsumer) { - int32_t code = 0; +int32_t tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType, char *topic, SCMSubscribeReq *subscribe, + SMqConsumerObj **ppConsumer) { + int32_t code = 0; SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj)); if (pConsumer == NULL) { code = terrno; @@ -294,30 +294,30 @@ int32_t tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType, pConsumer->createTime = taosGetTimestampMs(); pConsumer->updateType = updateType; - if (updateType == CONSUMER_ADD_REB){ + if (updateType == CONSUMER_ADD_REB) { pConsumer->rebNewTopics = taosArrayInit(0, sizeof(void *)); - if(pConsumer->rebNewTopics == NULL){ + if (pConsumer->rebNewTopics == NULL) { code = terrno; goto END; } - char* topicTmp = taosStrdup(topic); + char *topicTmp = taosStrdup(topic); if (taosArrayPush(pConsumer->rebNewTopics, &topicTmp) == NULL) { code = terrno; goto END; } - }else if (updateType == CONSUMER_REMOVE_REB) { + } else if (updateType == CONSUMER_REMOVE_REB) { pConsumer->rebRemovedTopics = taosArrayInit(0, sizeof(void *)); - if(pConsumer->rebRemovedTopics == NULL){ + if (pConsumer->rebRemovedTopics == NULL) { code = terrno; goto END; } - char* topicTmp = taosStrdup(topic); + char *topicTmp = taosStrdup(topic); if (taosArrayPush(pConsumer->rebRemovedTopics, &topicTmp) == NULL) { code = terrno; goto END; } - }else if (updateType == CONSUMER_INSERT_SUB){ + } else if (updateType == CONSUMER_INSERT_SUB) { tstrncpy(pConsumer->clientId, subscribe->clientId, tListLen(pConsumer->clientId)); pConsumer->withTbName = subscribe->withTbName; pConsumer->autoCommit = subscribe->autoCommit; @@ -329,13 +329,13 @@ int32_t tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType, tstrncpy(pConsumer->fqdn, subscribe->fqdn, TSDB_FQDN_LEN); pConsumer->rebNewTopics = taosArrayDup(subscribe->topicNames, topicNameDup); - if (pConsumer->rebNewTopics == NULL){ + if (pConsumer->rebNewTopics == NULL) { code = terrno; goto END; } pConsumer->assignedTopics = subscribe->topicNames; subscribe->topicNames = NULL; - }else if (updateType == CONSUMER_UPDATE_SUB){ + } else if (updateType == CONSUMER_UPDATE_SUB) { pConsumer->assignedTopics = subscribe->topicNames; subscribe->topicNames = NULL; } @@ -504,12 +504,12 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer, int8_t s buf = taosDecodeFixedI32(buf, &pConsumer->autoCommitInterval); buf = taosDecodeFixedI32(buf, &pConsumer->resetOffsetCfg); } - if (sver > 2){ + if (sver > 2) { buf = taosDecodeFixedI32(buf, &pConsumer->maxPollIntervalMs); buf = taosDecodeFixedI32(buf, &pConsumer->sessionTimeoutMs); buf = taosDecodeStringTo(buf, pConsumer->user); buf = taosDecodeStringTo(buf, pConsumer->fqdn); - } else{ + } else { pConsumer->maxPollIntervalMs = DEFAULT_MAX_POLL_INTERVAL; pConsumer->sessionTimeoutMs = DEFAULT_SESSION_TIMEOUT; } @@ -517,7 +517,7 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer, int8_t s return (void *)buf; } -int32_t tEncodeOffRows(void **buf, SArray *offsetRows){ +int32_t tEncodeOffRows(void **buf, SArray *offsetRows) { int32_t tlen = 0; int32_t szVgs = taosArrayGetSize(offsetRows); tlen += taosEncodeFixedI32(buf, szVgs); @@ -545,11 +545,10 @@ int32_t tEncodeSMqConsumerEp(void **buf, const SMqConsumerEp *pConsumerEp) { tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId); tlen += taosEncodeArray(buf, pConsumerEp->vgs, (FEncode)tEncodeSMqVgEp); - return tlen + tEncodeOffRows(buf, pConsumerEp->offsetRows); } -void *tDecodeOffRows(const void *buf, SArray **offsetRows, int8_t sver){ +void *tDecodeOffRows(const void *buf, SArray **offsetRows, int8_t sver) { int32_t szVgs = 0; buf = taosDecodeFixedI32(buf, &szVgs); if (szVgs > 0) { @@ -568,7 +567,7 @@ void *tDecodeOffRows(const void *buf, SArray **offsetRows, int8_t sver){ } else { // do nothing } - if(sver > 2){ + if (sver > 2) { buf = taosDecodeFixedI64(buf, &offRows->ever); } } @@ -587,7 +586,7 @@ void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp, int8_t s } int32_t tNewSubscribeObj(const char *key, SMqSubscribeObj **ppSub) { - int32_t code = 0; + int32_t code = 0; SMqSubscribeObj *pSubObj = taosMemoryCalloc(1, sizeof(SMqSubscribeObj)); MND_TMQ_NULL_CHECK(pSubObj); @@ -598,7 +597,7 @@ int32_t tNewSubscribeObj(const char *key, SMqSubscribeObj **ppSub) { MND_TMQ_NULL_CHECK(pSubObj->consumerHash); pSubObj->unassignedVgs = taosArrayInit(0, POINTER_BYTES); MND_TMQ_NULL_CHECK(pSubObj->unassignedVgs); - if (ppSub){ + if (ppSub) { *ppSub = pSubObj; } return code; @@ -609,7 +608,7 @@ END: } int32_t tCloneSubscribeObj(const SMqSubscribeObj *pSub, SMqSubscribeObj **ppSub) { - int32_t code = 0; + int32_t code = 0; SMqSubscribeObj *pSubNew = taosMemoryMalloc(sizeof(SMqSubscribeObj)); if (pSubNew == NULL) { code = terrno; @@ -730,6 +729,40 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub, int8_t sver) { return (void *)buf; } +SConfigObj *mndInitConfigObj(SConfigItem *pItem) { + SConfigObj *pObj = taosMemoryCalloc(1, sizeof(SConfigObj)); + if (pObj == NULL) { + return NULL; + } + strncpy(pObj->name, pItem->name, CFG_NAME_MAX_LEN); + pObj->dtype = pItem->dtype; + switch (pItem->dtype) { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + pObj->bval = pItem->bval; + break; + case CFG_DTYPE_INT32: + pObj->i32 = pItem->i32; + break; + case CFG_DTYPE_INT64: + pObj->i64 = pItem->i64; + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + pObj->fval = pItem->fval; + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + pObj->str = pItem->str; + break; + } + return pObj; +} + // SMqSubActionLogEntry *tCloneSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) { // SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry)); // if (pEntryNew == NULL) return NULL; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 39de235920..0ceb020ceb 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -612,6 +612,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-snode", mndInitSnode, mndCleanupSnode)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-anode", mndInitAnode, mndCleanupAnode)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-arbgroup", mndInitArbGroup, mndCleanupArbGroup)); + TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-config", mndInitConfig, NULL)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-grant", mndInitGrant, mndCleanupGrant)); @@ -638,7 +639,6 @@ static int32_t mndInitSteps(SMnode *pMnode) { TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-query", mndInitQuery, mndCleanupQuery)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-sync", mndInitSync, mndCleanupSync)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-telem", mndInitTelem, mndCleanupTelem)); - TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-config", mndInitConfig, NULL)); return 0; } diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 64cdcbee9e..1a013de1b6 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -410,7 +410,7 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { code = sdbWriteWithoutFree(pSdb, pRaw); if (code != 0) { - mError("failed to read sdb file:%s since %s", file, terrstr()); + mError("failed to exec sdbWrite while read sdb file:%s since %s", file, terrstr()); goto _OVER; } } diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index b159190119..17804804dc 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -261,6 +261,9 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * } int32_t sdbWriteWithoutFree(SSdb *pSdb, SSdbRaw *pRaw) { + if (pRaw->type == SDB_CFG) { + mTrace("sdb write cfg"); + } SHashObj *hash = sdbGetHash(pSdb, pRaw->type); if (hash == NULL) return terrno; diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index ca3b123472..f4b158075d 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -404,46 +404,6 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy TAOS_RETURN(code); } -int32_t cfgUpdateItem(SConfigItem *pItem, SConfigItem *newItem) { - int32_t code = TSDB_CODE_SUCCESS; - if (pItem == NULL || newItem == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); - } - - switch (pItem->dtype) { - case CFG_DTYPE_BOOL: { - pItem->bval = newItem->bval; - break; - } - case CFG_DTYPE_INT32: { - pItem->i32 = newItem->i32; - break; - } - case CFG_DTYPE_INT64: { - pItem->i64 = newItem->i64; - break; - } - case CFG_DTYPE_FLOAT: - case CFG_DTYPE_DOUBLE: { - pItem->fval = newItem->fval; - break; - } - case CFG_DTYPE_DIR: - case CFG_DTYPE_TIMEZONE: - case CFG_DTYPE_CHARSET: - case CFG_DTYPE_LOCALE: - case CFG_DTYPE_NONE: - case CFG_DTYPE_STRING: { - pItem->str = newItem->str; - break; - } - default: - code = TSDB_CODE_INVALID_CFG; - break; - } - - TAOS_RETURN(code); -} SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName) { if (pCfg == NULL) return NULL; From 6b744211ff3319fb1e193ad1e4a3f69a63a2c506 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 21 Nov 2024 14:30:28 +0800 Subject: [PATCH 22/99] Add alter dyn code. --- include/common/tmsg.h | 5 +- source/dnode/mnode/impl/inc/mndDef.h | 2 + source/dnode/mnode/impl/src/mndConfig.c | 227 +++++++++++++++++++++++- source/dnode/mnode/impl/src/mndDef.c | 65 +++++++ source/dnode/mnode/impl/src/mndDnode.c | 171 ------------------ 5 files changed, 293 insertions(+), 177 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index abe6465627..5f1d151a77 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2416,8 +2416,9 @@ int32_t tDeserializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq void tFreeSMCfgDnodeReq(SMCfgDnodeReq* pReq); typedef struct { - char config[TSDB_DNODE_CONFIG_LEN]; - char value[TSDB_DNODE_VALUE_LEN]; + char config[TSDB_DNODE_CONFIG_LEN]; + char value[TSDB_DNODE_VALUE_LEN]; + int32_t version; } SDCfgDnodeReq; int32_t tSerializeSDCfgDnodeReq(void* buf, int32_t bufLen, SDCfgDnodeReq* pReq); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 8770704211..49a96cf328 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -337,6 +337,8 @@ typedef struct { } SConfigObj; SConfigObj* mndInitConfigObj(SConfigItem* pItem); +SConfigObj* mndInitConfigVersion(); +int32_t mndUpdateObj(SConfigObj* pObj, const char* name, char* value); typedef struct { int32_t maxUsers; diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index fbe44759fd..9e68747878 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -14,14 +14,22 @@ */ #define _DEFAULT_SOURCE +#include "audit.h" #include "mndConfig.h" #include "mndDnode.h" +#include "mndPrivilege.h" #include "mndTrans.h" +#include "mndUser.h" +#include "tutil.h" #define CFG_VER_NUMBER 1 #define CFG_RESERVE_SIZE 63 +static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pInMCfgReq, int32_t optLen, int32_t *pOutValue); static int32_t cfgUpdateItem(SConfigItem *pItem, SConfigObj *obj); +static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue) static int32_t + mndProcessConfigDnodeReq(SRpcMsg *pReq); +static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessConfigReq(SRpcMsg *pReq); static int32_t mndInitWriteCfg(SMnode *pMnode); static int32_t mndInitReadCfg(SMnode *pMnode); @@ -41,6 +49,8 @@ int32_t mndInitConfig(SMnode *pMnode) { .prepareFp = (SdbPrepareFp)mndCfgActionPrepare}; mndSetMsgHandle(pMnode, TDMT_MND_CONFIG, mndProcessConfigReq); + mndSetMsgHandle(pMnode, TDMT_MND_CONFIG_DNODE, mndProcessConfigDnodeReq); + mndSetMsgHandle(pMnode, TDMT_DND_CONFIG_DNODE_RSP, mndProcessConfigDnodeRsp); return sdbSetTable(pMnode->pSdb, table); } @@ -83,8 +93,12 @@ SSdbRaw *mnCfgActionEncode(SConfigObj *obj) { case CFG_DTYPE_LOCALE: case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: - SDB_SET_INT32(pRaw, dataPos, strlen(obj->str), _OVER) - SDB_SET_BINARY(pRaw, dataPos, obj->str, strlen(obj->str), _OVER) + if (obj->str != NULL) { + SDB_SET_INT32(pRaw, dataPos, strlen(obj->str), _OVER) + SDB_SET_BINARY(pRaw, dataPos, obj->str, strlen(obj->str), _OVER) + } else { + SDB_SET_INT32(pRaw, dataPos, 0, _OVER) + } break; } @@ -149,7 +163,10 @@ SSdbRow *mndCfgActionDecode(SSdbRaw *pRaw) { case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: SDB_GET_INT32(pRaw, dataPos, &len, _OVER) - SDB_GET_BINARY(pRaw, dataPos, obj->str, len, _OVER) + if (len > 0) { + obj->str = taosMemoryMalloc(len + 1); + SDB_GET_BINARY(pRaw, dataPos, obj->str, len, _OVER) + } break; } terrno = TSDB_CODE_SUCCESS; @@ -337,7 +354,10 @@ int32_t cfgUpdateItem(SConfigItem *pItem, SConfigObj *obj) { case CFG_DTYPE_LOCALE: case CFG_DTYPE_NONE: case CFG_DTYPE_STRING: { - pItem->str = obj->str; + if (obj->str != NULL) { + taosMemoryFree(pItem->str); + pItem->str = obj->str; + } break; } default: @@ -346,4 +366,203 @@ int32_t cfgUpdateItem(SConfigItem *pItem, SConfigObj *obj) { } TAOS_RETURN(code); +} + +static int32_t mndMCfg2DCfg(SMCfgDnodeReq *pMCfgReq, SDCfgDnodeReq *pDCfgReq) { + int32_t code = 0; + char *p = pMCfgReq->config; + while (*p) { + if (*p == ' ') { + break; + } + p++; + } + + size_t optLen = p - pMCfgReq->config; + (void)strncpy(pDCfgReq->config, pMCfgReq->config, optLen); + pDCfgReq->config[optLen] = 0; + + if (' ' == pMCfgReq->config[optLen]) { + // 'key value' + if (strlen(pMCfgReq->value) != 0) goto _err; + (void)strcpy(pDCfgReq->value, p + 1); + } else { + // 'key' 'value' + if (strlen(pMCfgReq->value) == 0) goto _err; + (void)strcpy(pDCfgReq->value, pMCfgReq->value); + } + + TAOS_RETURN(code); + +_err: + mError("dnode:%d, failed to config since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config); + code = TSDB_CODE_INVALID_CFG; + TAOS_RETURN(code); +} + +static int32_t mndSendCfgDnodeReq(SMnode *pMnode, int32_t dnodeId, SDCfgDnodeReq *pDcfgReq) { + int32_t code = -1; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + while (1) { + SDnodeObj *pDnode = NULL; + pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); + if (pIter == NULL) break; + + if (pDnode->id == dnodeId || dnodeId == -1 || dnodeId == 0) { + SEpSet epSet = mndGetDnodeEpset(pDnode); + int32_t bufLen = tSerializeSDCfgDnodeReq(NULL, 0, pDcfgReq); + void *pBuf = rpcMallocCont(bufLen); + + if (pBuf != NULL) { + if ((bufLen = tSerializeSDCfgDnodeReq(pBuf, bufLen, pDcfgReq)) <= 0) { + code = bufLen; + return code; + } + mInfo("dnode:%d, send config req to dnode, config:%s value:%s", dnodeId, pDcfgReq->config, pDcfgReq->value); + SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen}; + code = tmsgSendReq(&epSet, &rpcMsg); + } + } + + sdbRelease(pSdb, pDnode); + } + + if (code == -1) { + code = TSDB_CODE_MND_DNODE_NOT_EXIST; + } + TAOS_RETURN(code); +} + +static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { + int32_t code = 0; + int32_t lino = -1; + SMnode *pMnode = pReq->info.node; + SMCfgDnodeReq cfgReq = {0}; + TAOS_CHECK_RETURN(tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq)); + int8_t updateIpWhiteList = 0; + mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value); + if ((code = mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE)) != 0) { + tFreeSMCfgDnodeReq(&cfgReq); + TAOS_RETURN(code); + } + + SDCfgDnodeReq dcfgReq = {0}; + if (strcasecmp(cfgReq.config, "resetlog") == 0) { + (void)strcpy(dcfgReq.config, "resetlog"); +#ifdef TD_ENTERPRISE + } else if (strncasecmp(cfgReq.config, "s3blocksize", 11) == 0) { + int32_t optLen = strlen("s3blocksize"); + int32_t flag = -1; + int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); + if (code < 0) return code; + + if (flag > 1024 * 1024 || (flag > -1 && flag < 1024) || flag < -1) { + mError("dnode:%d, failed to config s3blocksize since value:%d. Valid range: -1 or [1024, 1024 * 1024]", + cfgReq.dnodeId, flag); + code = TSDB_CODE_INVALID_CFG; + tFreeSMCfgDnodeReq(&cfgReq); + TAOS_RETURN(code); + } + + strcpy(dcfgReq.config, "s3blocksize"); + snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); +#endif + } else { + TAOS_CHECK_GOTO(mndMCfg2DCfg(&cfgReq, &dcfgReq), &lino, _err_out); + if (strlen(dcfgReq.config) > TSDB_DNODE_CONFIG_LEN) { + mError("dnode:%d, failed to config since config is too long", cfgReq.dnodeId); + code = TSDB_CODE_INVALID_CFG; + goto _err_out; + } + if (strncasecmp(dcfgReq.config, "enableWhiteList", strlen("enableWhiteList")) == 0) { + updateIpWhiteList = 1; + } + + TAOS_CHECK_GOTO(cfgCheckRangeForDynUpdate(taosGetCfg(), dcfgReq.config, dcfgReq.value, true), &lino, _err_out); + } + mndConfigUpdateTrans(pMnode, cfgReq.config, cfgReq.value); + { // audit + char obj[50] = {0}; + (void)sprintf(obj, "%d", cfgReq.dnodeId); + + auditRecord(pReq, pMnode->clusterId, "alterDnode", obj, "", cfgReq.sql, cfgReq.sqlLen); + } + + tFreeSMCfgDnodeReq(&cfgReq); + + code = mndSendCfgDnodeReq(pMnode, cfgReq.dnodeId, &dcfgReq); + + // dont care suss or succ; + if (updateIpWhiteList) mndRefreshUserIpWhiteList(pMnode); + TAOS_RETURN(code); + +_err_out: + tFreeSMCfgDnodeReq(&cfgReq); + TAOS_RETURN(code); +} + +static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp) { + mInfo("config rsp from dnode"); + return 0; +} + +// get int32_t value from 'SMCfgDnodeReq' +static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t optLen, int32_t *pOutValue) { + int32_t code = 0; + if (' ' != pMCfgReq->config[optLen] && 0 != pMCfgReq->config[optLen]) { + goto _err; + } + + if (' ' == pMCfgReq->config[optLen]) { + // 'key value' + if (strlen(pMCfgReq->value) != 0) goto _err; + *pOutValue = atoi(pMCfgReq->config + optLen + 1); + } else { + // 'key' 'value' + if (strlen(pMCfgReq->value) == 0) goto _err; + *pOutValue = atoi(pMCfgReq->value); + } + + TAOS_RETURN(code); + +_err: + mError("dnode:%d, failed to config since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config); + code = TSDB_CODE_INVALID_CFG; + TAOS_RETURN(code); +} + +// TODO(beryl):add more log. +static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue) { + int32_t code = -1; + int32_t lino = -1; + SConfigObj *pVersion = mndInitConfigVersion(); + if (pVersion == NULL) { + code = terrno; + goto _OVER; + } + SConfigObj *pObj = sdbAcquire(pMnode->pSdb, SDB_CFG, name); + if (pObj == NULL) { + mWarn("failed to acquire mnd config:%s while update config, since %s", name, terrstr()); + code = terrno; + goto _OVER; + } + TAOS_CHECK_GOTO(mndUpdateObj(pObj, name, pValue), &lino, _OVER); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_ARBGROUP, NULL, "update-config"); + if (pTrans == NULL) { + if (terrno != 0) code = terrno; + goto _OVER; + } + mInfo("trans:%d, used to update config:%s to value:%s", pTrans->id, pObj->name, pObj->str); + TAOS_CHECK_GOTO(mndSetCreateConfigCommitLogs(pTrans, pVersion), &lino, _OVER); + TAOS_CHECK_GOTO(mndSetCreateConfigCommitLogs(pTrans, pObj), &lino, _OVER); + if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER; + code = 0; +_OVER: + if (code != 0) { + --tsmmConfigVersion; + mError("failed to update config:%s to value:%s, since %s", name, pValue, tstrerror(code)); + } + mndTransDrop(pTrans); + return code; } \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index ad2c3f3b60..0ed136e35b 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -16,6 +16,7 @@ #include "mndConsumer.h" #include "mndDef.h" #include "taoserror.h" +#include "tunit.h" static void *freeStreamTasks(SArray *pTaskLevel); @@ -763,6 +764,70 @@ SConfigObj *mndInitConfigObj(SConfigItem *pItem) { return pObj; } +int32_t mndUpdateObj(SConfigObj *pObj, const char *name, char *value) { + int32_t code = 0; + + switch (pObj->dtype) { + case CFG_DTYPE_BOOL: { + bool tmp = false; + if (strcasecmp(value, "true") == 0) { + tmp = true; + } + if (atoi(value) > 0) { + tmp = true; + } + pObj->bval = tmp; + break; + } + case CFG_DTYPE_INT32: { + int32_t ival; + TAOS_CHECK_RETURN(taosStrHumanToInt32(value, &ival)); + pObj->i32 = ival; + break; + } + case CFG_DTYPE_INT64: { + int64_t ival; + TAOS_CHECK_RETURN(taosStrHumanToInt64(value, &ival)); + pObj->i64 = ival; + break; + } + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: { + float dval = 0; + TAOS_CHECK_RETURN(parseCfgReal(value, &dval)); + pObj->fval = dval; + break; + } + case CFG_DTYPE_DIR: + case CFG_DTYPE_TIMEZONE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_STRING: { + char *tmp = taosStrdup(value); + taosMemoryFreeClear(pObj->str); + pObj->str = tmp; + break; + } + + case CFG_DTYPE_NONE: + default: + code = TSDB_CODE_INVALID_CFG; + break; + } + return code; +} + +SConfigObj *mndInitConfigVersion() { + SConfigObj *pObj = taosMemoryCalloc(1, sizeof(SConfigObj)); + if (pObj == NULL) { + return NULL; + } + strncpy(pObj->name, "tsmmConfigVersion", CFG_NAME_MAX_LEN); + pObj->dtype = CFG_DTYPE_INT32; + pObj->i32 = ++tsmmConfigVersion; + return pObj; +} + // SMqSubActionLogEntry *tCloneSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) { // SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry)); // if (pEntryNew == NULL) return NULL; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index b4036e328e..fced2886ba 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -81,8 +81,6 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq); static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq); static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq); -static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq); -static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessStatusReq(SRpcMsg *pReq); static int32_t mndProcessNotifyReq(SRpcMsg *pReq); static int32_t mndProcessRestoreDnodeReq(SRpcMsg *pReq); @@ -97,8 +95,6 @@ static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter); static int32_t mndRetrieveDnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter); -static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pInMCfgReq, int32_t optLen, int32_t *pOutValue); - #ifdef _GRANT int32_t mndUpdClusterInfo(SRpcMsg *pReq); #else @@ -119,8 +115,6 @@ int32_t mndInitDnode(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_CREATE_DNODE, mndProcessCreateDnodeReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_DNODE, mndProcessDropDnodeReq); - mndSetMsgHandle(pMnode, TDMT_MND_CONFIG_DNODE, mndProcessConfigDnodeReq); - mndSetMsgHandle(pMnode, TDMT_DND_CONFIG_DNODE_RSP, mndProcessConfigDnodeRsp); mndSetMsgHandle(pMnode, TDMT_MND_STATUS, mndProcessStatusReq); mndSetMsgHandle(pMnode, TDMT_MND_NOTIFY, mndProcessNotifyReq); mndSetMsgHandle(pMnode, TDMT_MND_DNODE_LIST, mndProcessDnodeListReq); @@ -922,8 +916,6 @@ _OVER: return mndUpdClusterInfo(pReq); } - - static int32_t mndProcessNotifyReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SNotifyReq notifyReq = {0}; @@ -1441,144 +1433,6 @@ _OVER: TAOS_RETURN(code); } -static int32_t mndMCfg2DCfg(SMCfgDnodeReq *pMCfgReq, SDCfgDnodeReq *pDCfgReq) { - int32_t code = 0; - char *p = pMCfgReq->config; - while (*p) { - if (*p == ' ') { - break; - } - p++; - } - - size_t optLen = p - pMCfgReq->config; - (void)strncpy(pDCfgReq->config, pMCfgReq->config, optLen); - pDCfgReq->config[optLen] = 0; - - if (' ' == pMCfgReq->config[optLen]) { - // 'key value' - if (strlen(pMCfgReq->value) != 0) goto _err; - (void)strcpy(pDCfgReq->value, p + 1); - } else { - // 'key' 'value' - if (strlen(pMCfgReq->value) == 0) goto _err; - (void)strcpy(pDCfgReq->value, pMCfgReq->value); - } - - TAOS_RETURN(code); - -_err: - mError("dnode:%d, failed to config since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config); - code = TSDB_CODE_INVALID_CFG; - TAOS_RETURN(code); -} - -static int32_t mndSendCfgDnodeReq(SMnode *pMnode, int32_t dnodeId, SDCfgDnodeReq *pDcfgReq) { - int32_t code = -1; - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; - while (1) { - SDnodeObj *pDnode = NULL; - pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); - if (pIter == NULL) break; - - if (pDnode->id == dnodeId || dnodeId == -1 || dnodeId == 0) { - SEpSet epSet = mndGetDnodeEpset(pDnode); - int32_t bufLen = tSerializeSDCfgDnodeReq(NULL, 0, pDcfgReq); - void *pBuf = rpcMallocCont(bufLen); - - if (pBuf != NULL) { - if ((bufLen = tSerializeSDCfgDnodeReq(pBuf, bufLen, pDcfgReq)) <= 0) { - code = bufLen; - return code; - } - mInfo("dnode:%d, send config req to dnode, config:%s value:%s", dnodeId, pDcfgReq->config, pDcfgReq->value); - SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen}; - code = tmsgSendReq(&epSet, &rpcMsg); - } - } - - sdbRelease(pSdb, pDnode); - } - - if (code == -1) { - code = TSDB_CODE_MND_DNODE_NOT_EXIST; - } - TAOS_RETURN(code); -} - -static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { - int32_t code = 0; - SMnode *pMnode = pReq->info.node; - SMCfgDnodeReq cfgReq = {0}; - TAOS_CHECK_RETURN(tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq)); - int8_t updateIpWhiteList = 0; - mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value); - if ((code = mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE)) != 0) { - tFreeSMCfgDnodeReq(&cfgReq); - TAOS_RETURN(code); - } - - SDCfgDnodeReq dcfgReq = {0}; - if (strcasecmp(cfgReq.config, "resetlog") == 0) { - (void)strcpy(dcfgReq.config, "resetlog"); -#ifdef TD_ENTERPRISE - } else if (strncasecmp(cfgReq.config, "s3blocksize", 11) == 0) { - int32_t optLen = strlen("s3blocksize"); - int32_t flag = -1; - int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); - if (code < 0) return code; - - if (flag > 1024 * 1024 || (flag > -1 && flag < 1024) || flag < -1) { - mError("dnode:%d, failed to config s3blocksize since value:%d. Valid range: -1 or [1024, 1024 * 1024]", - cfgReq.dnodeId, flag); - code = TSDB_CODE_INVALID_CFG; - tFreeSMCfgDnodeReq(&cfgReq); - TAOS_RETURN(code); - } - - strcpy(dcfgReq.config, "s3blocksize"); - snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); -#endif - } else { - TAOS_CHECK_GOTO(mndMCfg2DCfg(&cfgReq, &dcfgReq), NULL, _err_out); - if (strlen(dcfgReq.config) > TSDB_DNODE_CONFIG_LEN) { - mError("dnode:%d, failed to config since config is too long", cfgReq.dnodeId); - code = TSDB_CODE_INVALID_CFG; - goto _err_out; - } - if (strncasecmp(dcfgReq.config, "enableWhiteList", strlen("enableWhiteList")) == 0) { - updateIpWhiteList = 1; - } - - TAOS_CHECK_GOTO(cfgCheckRangeForDynUpdate(taosGetCfg(), dcfgReq.config, dcfgReq.value, true), NULL, _err_out); - } - - { // audit - char obj[50] = {0}; - (void)sprintf(obj, "%d", cfgReq.dnodeId); - - auditRecord(pReq, pMnode->clusterId, "alterDnode", obj, "", cfgReq.sql, cfgReq.sqlLen); - } - - tFreeSMCfgDnodeReq(&cfgReq); - - code = mndSendCfgDnodeReq(pMnode, cfgReq.dnodeId, &dcfgReq); - - // dont care suss or succ; - if (updateIpWhiteList) mndRefreshUserIpWhiteList(pMnode); - TAOS_RETURN(code); - -_err_out: - tFreeSMCfgDnodeReq(&cfgReq); - TAOS_RETURN(code); -} - -static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp) { - mInfo("config rsp from dnode"); - return 0; -} - static int32_t mndProcessCreateEncryptKeyReqImpl(SRpcMsg *pReq, int32_t dnodeId, SDCfgDnodeReq *pDcfgReq) { int32_t code = 0; SMnode *pMnode = pReq->info.node; @@ -1874,31 +1728,6 @@ static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter) { sdbCancelFetchByType(pSdb, pIter, SDB_DNODE); } -// get int32_t value from 'SMCfgDnodeReq' -static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t optLen, int32_t *pOutValue) { - int32_t code = 0; - if (' ' != pMCfgReq->config[optLen] && 0 != pMCfgReq->config[optLen]) { - goto _err; - } - - if (' ' == pMCfgReq->config[optLen]) { - // 'key value' - if (strlen(pMCfgReq->value) != 0) goto _err; - *pOutValue = atoi(pMCfgReq->config + optLen + 1); - } else { - // 'key' 'value' - if (strlen(pMCfgReq->value) == 0) goto _err; - *pOutValue = atoi(pMCfgReq->value); - } - - TAOS_RETURN(code); - -_err: - mError("dnode:%d, failed to config since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config); - code = TSDB_CODE_INVALID_CFG; - TAOS_RETURN(code); -} - SArray *mndGetAllDnodeFqdns(SMnode *pMnode) { SDnodeObj *pObj = NULL; void *pIter = NULL; From f46037870dc22692ccaa56dea1a32d8facd52bb0 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 21 Nov 2024 15:34:32 +0800 Subject: [PATCH 23/99] Add version control and persist. --- include/util/tconfig.h | 10 +- source/common/src/tglobal.c | 6 +- source/common/src/tmsg.c | 2 + source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 21 ++-- source/dnode/mnode/impl/src/mndConfig.c | 18 ++-- source/util/src/tconfig.c | 109 +++++++++++--------- 6 files changed, 98 insertions(+), 68 deletions(-) diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 98e0df9655..4ba9a9c93e 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -118,6 +118,8 @@ void cfgCleanup(SConfig *pCfg); int32_t cfgGetSize(SConfig *pCfg); SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName); int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype, bool lock); +int32_t cfgGetAndSetItem(SConfig *pCfg, SConfigItem *pItem, const char *name, const char *value, ECfgSrcType stype, + bool lock); int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer); int32_t cfgCreateIter(SConfig *pConf, SConfigIter **ppIter); @@ -148,11 +150,11 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump); void cfgDumpCfgS3(SConfig *pCfg, bool tsc, bool dump); int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl); -SArray *getLocalCfg(SConfig *pCfg); -SArray *getGlobalCfg(SConfig *pCfg); +SArray *taosGetLocalCfg(SConfig *pCfg); +SArray *taosGetGlobalCfg(SConfig *pCfg); -void setLocalCfg(SConfig *pCfg, SArray *pArray); -void setGlobalCfg(SConfig *pCfg, SArray *pArray); +void taosSetLocalCfg(SConfig *pCfg, SArray *pArray); +void taosSetGlobalCfg(SConfig *pCfg, SArray *pArray); #ifdef __cplusplus } #endif diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c8b214ecfa..50fab2d3f4 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1969,10 +1969,10 @@ int32_t readCfgFile(const char *path, bool isGlobal) { char filename[CONFIG_FILE_LEN] = {0}; SArray *array = NULL; if (isGlobal) { - array = getGlobalCfg(tsCfg); + array = taosGetGlobalCfg(tsCfg); snprintf(filename, sizeof(filename), "%s%sconfig%sglobal.json", path, TD_DIRSEP, TD_DIRSEP); } else { - array = getLocalCfg(tsCfg); + array = taosGetLocalCfg(tsCfg); snprintf(filename, sizeof(filename), "%s%sconfig%slocal.json", path, TD_DIRSEP, TD_DIRSEP); } @@ -2802,7 +2802,7 @@ int32_t persistLocalConfig(const char *path) { TAOS_RETURN(code); } char *serialized = NULL; - code = localConfigSerialize(getLocalCfg(tsCfg), &serialized); + code = localConfigSerialize(taosGetLocalCfg(tsCfg), &serialized); if (code != TSDB_CODE_SUCCESS) { uError("failed to serialize local config since %s", tstrerror(code)); TAOS_RETURN(code); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 83577403aa..989078d20d 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3243,6 +3243,7 @@ int32_t tSerializeSDCfgDnodeReq(void *buf, int32_t bufLen, SDCfgDnodeReq *pReq) tEncoderInit(&encoder, buf, bufLen); TAOS_CHECK_EXIT(tStartEncode(&encoder)); + TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->version)); TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->config)); TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->value)); tEndEncode(&encoder); @@ -3264,6 +3265,7 @@ int32_t tDeserializeSDCfgDnodeReq(void *buf, int32_t bufLen, SDCfgDnodeReq *pReq tDecoderInit(&decoder, buf, bufLen); TAOS_CHECK_EXIT(tStartDecode(&decoder)); + TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->version)); TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->config)); TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->value)); tEndDecode(&decoder); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 15176f58a4..7301923873 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -304,7 +304,7 @@ static void dmProcessConfigRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { tDeserializeSConfigRsp(pRsp->pCont, pRsp->contLen, &configRsp) == 0) { if (configRsp.forceReadConfig) { if (configRsp.isConifgVerified) { - persistGlobalConfig(getGlobalCfg(tsCfg), pMgmt->path, configRsp.cver); + persistGlobalConfig(taosGetGlobalCfg(tsCfg), pMgmt->path, configRsp.cver); } else { // log the difference configurations printConfigNotMatch(configRsp.array); @@ -330,7 +330,7 @@ void dmSendConfigReq(SDnodeMgmt *pMgmt) { req.cver = tsdmConfigVersion; req.forceReadConfig = tsForceReadConfig; - req.array = getGlobalCfg(tsCfg); + req.array = taosGetGlobalCfg(tsCfg); dDebug("send config req to mnode, configVersion:%d", req.cver); int32_t contLen = tSerializeSConfigReq(NULL, 0, &req); @@ -446,9 +446,9 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { dInfo("start to config, option:%s, value:%s", cfgReq.config, cfgReq.value); - SConfig *pCfg = taosGetCfg(); - - code = cfgSetItem(pCfg, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_CMD, true); + SConfig *pCfg = taosGetCfg(); + SConfigItem *pItem = NULL; + code = cfgGetAndSetItem(pCfg, pItem, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_CMD, true); if (code != 0) { if (strncasecmp(cfgReq.config, "resetlog", strlen("resetlog")) == 0) { code = 0; @@ -456,8 +456,17 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { return code; } } + TAOS_CHECK_RETURN(taosCfgDynamicOptions(pCfg, cfgReq.config, true)); + if (cfgReq.version > 0) { + tsdmConfigVersion = cfgReq.version; + } + if (pItem->category == CFG_CATEGORY_GLOBAL) { + persistGlobalConfig(taosGetGlobalCfg(pCfg), pMgmt->path, tsdmConfigVersion); + } else { + persistLocalConfig(pMgmt->path); + } - return taosCfgDynamicOptions(pCfg, cfgReq.config, true); + return TSDB_CODE_SUCCESS; } int32_t dmProcessCreateEncryptKeyReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index 9e68747878..a044f6e8c4 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -214,18 +214,18 @@ static int32_t mndProcessConfigReq(SRpcMsg *pReq) { configRsp.cver = tsmmConfigVersion; if (configRsp.forceReadConfig) { // compare config array from configReq with current config array - if (compareSConfigItemArrays(getGlobalCfg(tsCfg), configReq.array, diffArray)) { + if (compareSConfigItemArrays(taosGetGlobalCfg(tsCfg), configReq.array, diffArray)) { configRsp.array = diffArray; } else { configRsp.isConifgVerified = 1; taosArrayDestroy(diffArray); } } else { - configRsp.array = getGlobalCfg(tsCfg); + configRsp.array = taosGetGlobalCfg(tsCfg); if (configReq.cver == tsmmConfigVersion) { configRsp.isVersionVerified = 1; } else { - configRsp.array = getGlobalCfg(tsCfg); + configRsp.array = taosGetGlobalCfg(tsCfg); } } @@ -261,10 +261,10 @@ int32_t mndInitWriteCfg(SMnode *pMnode) { if ((code = mndSetCreateConfigCommitLogs(pTrans, &obj)) != 0) { mError("failed to init mnd config version, since %s", terrstr()); } - sz = taosArrayGetSize(getGlobalCfg(tsCfg)); + sz = taosArrayGetSize(taosGetGlobalCfg(tsCfg)); for (int i = 0; i < sz; ++i) { - SConfigItem *item = taosArrayGet(getGlobalCfg(tsCfg), i); + SConfigItem *item = taosArrayGet(taosGetGlobalCfg(tsCfg), i); SConfigObj *obj = mndInitConfigObj(item); if ((code = mndSetCreateConfigCommitLogs(pTrans, obj)) != 0) { mError("failed to init mnd config:%s, since %s", item->name, terrstr()); @@ -296,9 +296,9 @@ int32_t mndInitReadCfg(SMnode *pMnode) { sdbRelease(pMnode->pSdb, obj); } - sz = taosArrayGetSize(getGlobalCfg(tsCfg)); + sz = taosArrayGetSize(taosGetGlobalCfg(tsCfg)); for (int i = 0; i < sz; ++i) { - SConfigItem *item = taosArrayGet(getGlobalCfg(tsCfg), i); + SConfigItem *item = taosArrayGet(taosGetGlobalCfg(tsCfg), i); SConfigObj *newObj = sdbAcquire(pMnode->pSdb, SDB_CFG, item->name); if (newObj == NULL) { mInfo("failed to acquire mnd config:%s, since %s", item->name, terrstr()); @@ -481,7 +481,8 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { TAOS_CHECK_GOTO(cfgCheckRangeForDynUpdate(taosGetCfg(), dcfgReq.config, dcfgReq.value, true), &lino, _err_out); } - mndConfigUpdateTrans(pMnode, cfgReq.config, cfgReq.value); + // update config in sdb + TAOS_CHECK_GOTO(mndConfigUpdateTrans(pMnode, cfgReq.config, cfgReq.value), &lino, _err_out); { // audit char obj[50] = {0}; (void)sprintf(obj, "%d", cfgReq.dnodeId); @@ -491,6 +492,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { tFreeSMCfgDnodeReq(&cfgReq); + dcfgReq.version = tsmmConfigVersion; code = mndSendCfgDnodeReq(pMnode, cfgReq.dnodeId, &dcfgReq); // dont care suss or succ; diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index f4b158075d..f05c90f6a3 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -40,6 +40,7 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile); int32_t cfgLoadFromEnvVar(SConfig *pConfig); int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd); int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url); +int32_t cfgSetItemVal(SConfigItem *item, ECfgDataType dtype); extern char **environ; @@ -353,49 +354,7 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); } - switch (pItem->dtype) { - case CFG_DTYPE_BOOL: { - code = cfgSetBool(pItem, value, stype); - break; - } - case CFG_DTYPE_INT32: { - code = cfgSetInt32(pItem, value, stype); - break; - } - case CFG_DTYPE_INT64: { - code = cfgSetInt64(pItem, value, stype); - break; - } - case CFG_DTYPE_FLOAT: - case CFG_DTYPE_DOUBLE: { - code = cfgSetFloat(pItem, value, stype); - break; - } - case CFG_DTYPE_STRING: { - code = cfgSetString(pItem, value, stype); - break; - } - case CFG_DTYPE_DIR: { - code = cfgSetDir(pItem, value, stype); - break; - } - case CFG_DTYPE_TIMEZONE: { - code = cfgSetTimezone(pItem, value, stype); - break; - } - case CFG_DTYPE_CHARSET: { - code = doSetConf(pItem, value, stype); - break; - } - case CFG_DTYPE_LOCALE: { - code = doSetConf(pItem, value, stype); - break; - } - case CFG_DTYPE_NONE: - default: - code = TSDB_CODE_INVALID_CFG; - break; - } + TAOS_CHECK_RETURN(cfgSetItemVal(pItem, pItem->dtype)); if (lock) { (void)taosThreadMutexUnlock(&pCfg->lock); @@ -404,6 +363,62 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy TAOS_RETURN(code); } +int32_t cfgGetAndSetItem(SConfig *pCfg, SConfigItem *pItem, const char *name, const char *value, ECfgSrcType stype, + bool lock) { + // GRANT_CFG_SET; + int32_t code = TSDB_CODE_SUCCESS; + + if (lock) { + (void)taosThreadMutexLock(&pCfg->lock); + } + + pItem = cfgGetItem(pCfg, name); + if (pItem == NULL) { + (void)taosThreadMutexUnlock(&pCfg->lock); + TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); + } + + TAOS_CHECK_RETURN(cfgSetItemVal(pItem, pItem->dtype)); + + if (lock) { + (void)taosThreadMutexUnlock(&pCfg->lock); + } + + TAOS_RETURN(code); +} + +int32_t cfgSetItemVal(SConfigItem *item, ECfgDataType dtype) { + if (item == NULL) { + TAOS_RETURN(TSDB_CODE_INVALID_CFG); + } + item->dtype = dtype; + switch (dtype) { + case CFG_DTYPE_BOOL: + item->bval = false; + break; + case CFG_DTYPE_INT32: + item->i32 = 0; + break; + case CFG_DTYPE_INT64: + item->i64 = 0; + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + item->fval = 0; + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + item->str = NULL; + break; + default: + TAOS_RETURN(TSDB_CODE_INVALID_CFG); + } + + TAOS_RETURN(TSDB_CODE_SUCCESS); +} SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName) { if (pCfg == NULL) return NULL; @@ -1538,7 +1553,7 @@ void cfgDestroyIter(SConfigIter *pIter) { taosMemoryFree(pIter); } -SArray *getLocalCfg(SConfig *pCfg) { return pCfg->localArray; } -SArray *getGlobalCfg(SConfig *pCfg) { return pCfg->globalArray; } -void setLocalCfg(SConfig *pCfg, SArray *pArray) { pCfg->localArray = pArray; }; -void setGlobalCfg(SConfig *pCfg, SArray *pArray) { pCfg->globalArray = pArray; }; \ No newline at end of file +SArray *taosGetLocalCfg(SConfig *pCfg) { return pCfg->localArray; } +SArray *taosGetGlobalCfg(SConfig *pCfg) { return pCfg->globalArray; } +void taosSetLocalCfg(SConfig *pCfg, SArray *pArray) { pCfg->localArray = pArray; }; +void taosSetGlobalCfg(SConfig *pCfg, SArray *pArray) { pCfg->globalArray = pArray; }; \ No newline at end of file From 1089f323e4cf57f04277961f6435d357ee71af65 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 21 Nov 2024 16:43:58 +0800 Subject: [PATCH 24/99] Add logic to set configs while init dnode. --- source/common/src/tglobal.c | 8 ++++++-- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 9 +++++++-- source/dnode/mnode/impl/src/mndConfig.c | 4 ++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 50fab2d3f4..7cd05f7fb6 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -35,7 +35,6 @@ // GRANT_CFG_DECLARE; SConfig *tsCfg = NULL; - // cluster char tsFirst[TSDB_EP_LEN] = {0}; char tsSecond[TSDB_EP_LEN] = {0}; @@ -602,6 +601,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { (void)strcpy(defaultFqdn, "localhost"); } + TAOS_CHECK_RETURN( + cfgAddBool(pCfg, "forceReadConfig", tsForceReadConfig, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "firstEp", "", CFG_SCOPE_BOTH, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "secondEp", "", CFG_SCOPE_BOTH, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "fqdn", defaultFqdn, CFG_SCOPE_SERVER, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); @@ -1230,6 +1231,9 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "fqdn"); tstrncpy(tsLocalFqdn, pItem->str, TSDB_FQDN_LEN); + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "forceReadConfig"); + tsForceReadConfig = pItem->bval; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "serverPort"); tsServerPort = (uint16_t)pItem->i32; (void)snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); @@ -2060,7 +2064,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile tsCfg = NULL; TAOS_RETURN(code); } - tryLoadCfgFromDataDir(tsCfg); + TAOS_CHECK_GOTO(tryLoadCfgFromDataDir(tsCfg), &lino, _exit); if (tsc) { TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 7301923873..c106278009 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -286,7 +286,6 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { static void dmProcessConfigRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { const STraceId *trace = &pRsp->info.traceId; SConfigRsp configRsp = {0}; - dGTrace("status rsp received from mnode, statusSeq:%d code:0x%x", pMgmt->statusSeq, pRsp->code); if (pRsp->code != 0) { if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pMgmt->pData->dropped && pMgmt->pData->dnodeId > 0) { @@ -300,8 +299,10 @@ static void dmProcessConfigRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { (void)raise(SIGINT); } } else { + bool needUpdate = false; if (pRsp->pCont != NULL && pRsp->contLen > 0 && tDeserializeSConfigRsp(pRsp->pCont, pRsp->contLen, &configRsp) == 0) { + // Try to use cfg file in current dnode. if (configRsp.forceReadConfig) { if (configRsp.isConifgVerified) { persistGlobalConfig(taosGetGlobalCfg(tsCfg), pMgmt->path, configRsp.cver); @@ -311,11 +312,15 @@ static void dmProcessConfigRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { goto _exit; } } + // Try to use cfg from mnode sdb. if (!configRsp.isVersionVerified) { + needUpdate = true; persistGlobalConfig(configRsp.array, pMgmt->path, configRsp.cver); } } - setAllConfigs(tsCfg); + if (needUpdate) { + setAllConfigs(tsCfg); + } persistLocalConfig(pMgmt->path); tsConfigInited = 1; } diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index a044f6e8c4..e5e6a95c8d 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -27,8 +27,8 @@ static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pInMCfgReq, int32_t optLen, int32_t *pOutValue); static int32_t cfgUpdateItem(SConfigItem *pItem, SConfigObj *obj); -static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue) static int32_t - mndProcessConfigDnodeReq(SRpcMsg *pReq); +static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue); +static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq); static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessConfigReq(SRpcMsg *pReq); static int32_t mndInitWriteCfg(SMnode *pMnode); From db67245af0f0775f7eb51082d1b5702cd8ade0f4 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Fri, 22 Nov 2024 15:57:18 +0800 Subject: [PATCH 25/99] Add more dyn configs and fix vesion in mnd bug. --- include/util/tconfig.h | 4 +- source/common/src/tglobal.c | 58 +++++++++++++++- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- source/dnode/mnode/impl/src/mndConfig.c | 24 ++++--- source/libs/command/src/command.c | 6 +- source/util/src/tconfig.c | 75 ++++++++++++++------- 6 files changed, 129 insertions(+), 40 deletions(-) diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 4ba9a9c93e..cfccbc16c8 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -118,9 +118,9 @@ void cfgCleanup(SConfig *pCfg); int32_t cfgGetSize(SConfig *pCfg); SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName); int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype, bool lock); -int32_t cfgGetAndSetItem(SConfig *pCfg, SConfigItem *pItem, const char *name, const char *value, ECfgSrcType stype, +int32_t cfgGetAndSetItem(SConfig *pCfg, SConfigItem **ppItem, const char *name, const char *value, ECfgSrcType stype, bool lock); -int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer); +int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer, bool isUpdateAll); int32_t cfgCreateIter(SConfig *pConf, SConfigIter **ppIter); SConfigItem *cfgNextIter(SConfigIter *pIter); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 7cd05f7fb6..a45d3b59dc 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -45,7 +45,7 @@ uint16_t tsServerPort = 6030; int32_t tsVersion = 30000000; int32_t tsForceReadConfig = 0; int32_t tsdmConfigVersion = -1; -int32_t tsmmConfigVersion = 0; +int32_t tsmmConfigVersion = -1; int32_t tsConfigInited = 0; int32_t tsStatusInterval = 1; // second int32_t tsNumOfSupportVnodes = 256; @@ -2225,12 +2225,33 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"asynclog", &tsAsyncLog}, {"disableStream", &tsDisableStream}, {"enableWhiteList", &tsEnableWhiteList}, + {"statusInterval", &tsStatusInterval}, {"telemetryReporting", &tsEnableTelem}, {"monitor", &tsEnableMonitor}, {"monitorInterval", &tsMonitorInterval}, + {"monitorComp", &tsMonitorComp}, + {"monitorForceV2", &tsMonitorForceV2}, + {"monitorLogProtocol", &tsMonitorLogProtocol}, + {"monitorMaxLogs", &tsMonitorMaxLogs}, + {"audit", &tsEnableAudit}, + {"auditCreateTable", &tsEnableAuditCreateTable}, + {"auditInterval", &tsAuditInterval}, {"slowLogThreshold", &tsSlowLogThreshold}, + {"compressMsgSize", &tsCompressMsgSize}, + {"compressor", &tsCompressor}, + {"curRange", &tsCurRange}, + {"dPrecision", &tsDPrecision}, + {"fPrecision", &tsFPrecision}, + {"ifAdtFse", &tsIfAdtFse}, + {"maxRange", &tsMaxRange}, + {"maxTsmaNum", &tsMaxTsmaNum}, + {"queryRsmaTolerance", &tsQueryRsmaTolerance}, + {"countAlwaysReturnValue", &tsCountAlwaysReturnValue}, + {"uptimeInterval", &tsUptimeInterval}, {"slowLogThresholdTest", &tsSlowLogThresholdTest}, {"slowLogMaxLen", &tsSlowLogMaxLen}, + {"slowLogScope", &tsSlowLogScope}, + {"slowLogExceptDb", &tsSlowLogExceptDb}, {"mndSdbWriteDelta", &tsMndSdbWriteDelta}, {"minDiskFreeSize", &tsMinDiskFreeSize}, @@ -2238,9 +2259,28 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"randErrorDivisor", &tsRandErrDivisor}, {"randErrorScope", &tsRandErrScope}, {"syncLogBufferMemoryAllowed", &tsLogBufferMemoryAllowed}, + {"resolveFQDNRetryTime", &tsResolveFQDNRetryTime}, + {"syncElectInterval", &tsElectInterval}, + {"syncHeartbeatInterval", &tsHeartbeatInterval}, + {"syncHeartbeatTimeout", &tsHeartbeatTimeout}, + {"syncSnapReplMaxWaitN", &tsSnapReplMaxWaitN}, + {"walFsyncDataSizeLimit", &tsWalFsyncDataSizeLimit}, + + {"numOfCores", &tsNumOfCores}, + + {"enableCoreFile", &tsEnableCoreFile}, + + {"telemetryInterval", &tsTelemInterval}, + {"telemetryPort", &tsTelemPort}, {"cacheLazyLoadThreshold", &tsCacheLazyLoadThreshold}, {"checkpointInterval", &tsStreamCheckpointInterval}, + {"concurrentCheckpoint", &tsMaxConcurrentCheckpoint}, + + {"retentionSpeedLimitMB", &tsRetentionSpeedLimitMB}, + {"trimVDbIntervalSec", &tsTrimVDbIntervalSec}, + {"ttlChangeOnWrite", &tsTtlChangeOnWrite}, + {"keepAliveIdle", &tsKeepAliveIdle}, {"logKeepDays", &tsLogKeepDays}, {"maxStreamBackendCache", &tsMaxStreamBackendCache}, @@ -2256,15 +2296,31 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"ttlBatchDropNum", &tsTtlBatchDropNum}, {"ttlFlushThreshold", &tsTtlFlushThreshold}, {"ttlPushInterval", &tsTtlPushIntervalSec}, + {"ttlUnit", &tsTtlUnit}, + {"udf", &tsStartUdfd}, + {"udfdLdLibPath", &tsUdfdLdLibPath}, + {"udfdResFuncs", &tsUdfdResFuncs}, + {"s3Accesskey", &tsS3AccessKey}, + {"s3BucketName", &tsS3BucketName}, + {"s3Endpoint", &tsS3Endpoint}, {"s3MigrateIntervalSec", &tsS3MigrateIntervalSec}, {"s3MigrateEnabled", &tsS3MigrateEnabled}, //{"s3BlockSize", &tsS3BlockSize}, {"s3BlockCacheSize", &tsS3BlockCacheSize}, {"s3PageCacheSize", &tsS3PageCacheSize}, {"s3UploadDelaySec", &tsS3UploadDelaySec}, + {"mndSdbWriteDelta", &tsMndSdbWriteDelta}, + {"mndLogRetention", &tsMndLogRetention}, {"supportVnodes", &tsNumOfSupportVnodes}, {"experimental", &tsExperimental}, {"maxTsmaNum", &tsMaxTsmaNum}, + {"maxShellConns", &tsMaxShellConns}, + {"numOfRpcSessions", &tsNumOfRpcSessions}, + {"numOfRpcThreads", &tsNumOfRpcThreads}, + {"rpcQueueMemoryAllowed", &tsQueueMemoryAllowed}, + {"shellActivityTimer", &tsShellActivityTimer}, + {"timeToGetAvailableConn", &tsTimeToGetAvailableConn}, + {"tsReadTimeout", &tsReadTimeout}, {"safetyCheckLevel", &tsSafetyCheckLevel}}; if ((code = taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true)) != TSDB_CODE_SUCCESS) { diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index c106278009..105f597d22 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -453,7 +453,7 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { SConfig *pCfg = taosGetCfg(); SConfigItem *pItem = NULL; - code = cfgGetAndSetItem(pCfg, pItem, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_CMD, true); + code = cfgGetAndSetItem(pCfg, &pItem, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_CMD, true); if (code != 0) { if (strncasecmp(cfgReq.config, "resetlog", strlen("resetlog")) == 0) { code = 0; diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index e5e6a95c8d..a8880ce94c 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -25,6 +25,11 @@ #define CFG_VER_NUMBER 1 #define CFG_RESERVE_SIZE 63 +enum CfgAlterType { + CFG_ALTER_DNODE, + CFG_ALTER_ALL_DNODES, +}; + static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pInMCfgReq, int32_t optLen, int32_t *pOutValue); static int32_t cfgUpdateItem(SConfigItem *pItem, SConfigObj *obj); static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue); @@ -249,16 +254,15 @@ int32_t mndInitWriteCfg(SMnode *pMnode) { int code = -1; size_t sz = 0; - SConfigObj obj = {0}; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "init-write-config"); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "init-write-config"); if (pTrans == NULL) { mError("failed to init write cfg in create trans, since %s", terrstr()); goto _OVER; } // encode mnd config version - obj = (SConfigObj){.name = "tsmmConfigVersion", .dtype = CFG_DTYPE_INT32, .i32 = tsmmConfigVersion}; - if ((code = mndSetCreateConfigCommitLogs(pTrans, &obj)) != 0) { + SConfigObj *obj = mndInitConfigVersion(); + if ((code = mndSetCreateConfigCommitLogs(pTrans, obj)) != 0) { mError("failed to init mnd config version, since %s", terrstr()); } sz = taosArrayGetSize(taosGetGlobalCfg(tsCfg)); @@ -479,9 +483,11 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { updateIpWhiteList = 1; } - TAOS_CHECK_GOTO(cfgCheckRangeForDynUpdate(taosGetCfg(), dcfgReq.config, dcfgReq.value, true), &lino, _err_out); + bool isUpdateAll = (cfgReq.dnodeId == 0 || cfgReq.dnodeId == -1) ? true : false; + TAOS_CHECK_GOTO(cfgCheckRangeForDynUpdate(taosGetCfg(), dcfgReq.config, dcfgReq.value, true, isUpdateAll), &lino, + _err_out); } - // update config in sdb + // Update config in sdb. TAOS_CHECK_GOTO(mndConfigUpdateTrans(pMnode, cfgReq.config, cfgReq.value), &lino, _err_out); { // audit char obj[50] = {0}; @@ -538,11 +544,13 @@ _err: static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue) { int32_t code = -1; int32_t lino = -1; - SConfigObj *pVersion = mndInitConfigVersion(); + SConfigObj *pVersion = sdbAcquire(pMnode->pSdb, SDB_CFG, "tsmmConfigVersion"); if (pVersion == NULL) { + mWarn("failed to acquire tsmmConfigVersion while update config, since %s", terrstr()); code = terrno; goto _OVER; } + pVersion->i32 = ++tsmmConfigVersion; SConfigObj *pObj = sdbAcquire(pMnode->pSdb, SDB_CFG, name); if (pObj == NULL) { mWarn("failed to acquire mnd config:%s while update config, since %s", name, terrstr()); @@ -555,7 +563,7 @@ static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pVal if (terrno != 0) code = terrno; goto _OVER; } - mInfo("trans:%d, used to update config:%s to value:%s", pTrans->id, pObj->name, pObj->str); + mInfo("trans:%d, used to update config:%s to value:%s", pTrans->id, name, pValue); TAOS_CHECK_GOTO(mndSetCreateConfigCommitLogs(pTrans, pVersion), &lino, _OVER); TAOS_CHECK_GOTO(mndSetCreateConfigCommitLogs(pTrans, pObj), &lino, _OVER); if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 6272ac7049..678d0844a8 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "command.h" #include "catalog.h" +#include "command.h" #include "commandInt.h" #include "scheduler.h" #include "systable.h" @@ -51,7 +51,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->numOfCols = htonl(numOfCols); int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, dataEncodeBufSize, numOfCols); - if(len < 0) { + if (len < 0) { taosMemoryFree(*pRsp); return terrno; } @@ -898,7 +898,7 @@ static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { goto _return; } - if (cfgCheckRangeForDynUpdate(tsCfg, pStmt->config, pStmt->value, false)) { + if (cfgCheckRangeForDynUpdate(tsCfg, pStmt->config, pStmt->value, false, false)) { return terrno; } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index f05c90f6a3..72dcd8f284 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -40,7 +40,7 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile); int32_t cfgLoadFromEnvVar(SConfig *pConfig); int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd); int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url); -int32_t cfgSetItemVal(SConfigItem *item, ECfgDataType dtype); +int32_t cfgSetItemVal(SConfigItem *pItem, const char *name, const char *value, ECfgSrcType stype); extern char **environ; @@ -354,7 +354,7 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); } - TAOS_CHECK_RETURN(cfgSetItemVal(pItem, pItem->dtype)); + TAOS_CHECK_RETURN(cfgSetItemVal(pItem, name, value, stype)); if (lock) { (void)taosThreadMutexUnlock(&pCfg->lock); @@ -363,7 +363,7 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy TAOS_RETURN(code); } -int32_t cfgGetAndSetItem(SConfig *pCfg, SConfigItem *pItem, const char *name, const char *value, ECfgSrcType stype, +int32_t cfgGetAndSetItem(SConfig *pCfg, SConfigItem **pItem, const char *name, const char *value, ECfgSrcType stype, bool lock) { // GRANT_CFG_SET; int32_t code = TSDB_CODE_SUCCESS; @@ -372,13 +372,13 @@ int32_t cfgGetAndSetItem(SConfig *pCfg, SConfigItem *pItem, const char *name, co (void)taosThreadMutexLock(&pCfg->lock); } - pItem = cfgGetItem(pCfg, name); + *pItem = cfgGetItem(pCfg, name); if (pItem == NULL) { (void)taosThreadMutexUnlock(&pCfg->lock); TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); } - TAOS_CHECK_RETURN(cfgSetItemVal(pItem, pItem->dtype)); + TAOS_CHECK_RETURN(cfgSetItemVal(*pItem, name, value, stype)); if (lock) { (void)taosThreadMutexUnlock(&pCfg->lock); @@ -387,34 +387,54 @@ int32_t cfgGetAndSetItem(SConfig *pCfg, SConfigItem *pItem, const char *name, co TAOS_RETURN(code); } -int32_t cfgSetItemVal(SConfigItem *item, ECfgDataType dtype) { - if (item == NULL) { +int32_t cfgSetItemVal(SConfigItem *pItem, const char *name, const char *value, ECfgSrcType stype) { + int32_t code = TSDB_CODE_SUCCESS; + + if (pItem == NULL) { TAOS_RETURN(TSDB_CODE_INVALID_CFG); } - item->dtype = dtype; - switch (dtype) { - case CFG_DTYPE_BOOL: - item->bval = false; + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: { + code = cfgSetBool(pItem, value, stype); break; - case CFG_DTYPE_INT32: - item->i32 = 0; + } + case CFG_DTYPE_INT32: { + code = cfgSetInt32(pItem, value, stype); break; - case CFG_DTYPE_INT64: - item->i64 = 0; + } + case CFG_DTYPE_INT64: { + code = cfgSetInt64(pItem, value, stype); break; + } case CFG_DTYPE_FLOAT: - case CFG_DTYPE_DOUBLE: - item->fval = 0; + case CFG_DTYPE_DOUBLE: { + code = cfgSetFloat(pItem, value, stype); break; - case CFG_DTYPE_STRING: - case CFG_DTYPE_DIR: - case CFG_DTYPE_LOCALE: - case CFG_DTYPE_CHARSET: - case CFG_DTYPE_TIMEZONE: - item->str = NULL; + } + case CFG_DTYPE_STRING: { + code = cfgSetString(pItem, value, stype); break; + } + case CFG_DTYPE_DIR: { + code = cfgSetDir(pItem, value, stype); + break; + } + case CFG_DTYPE_TIMEZONE: { + code = cfgSetTimezone(pItem, value, stype); + break; + } + case CFG_DTYPE_CHARSET: { + code = doSetConf(pItem, value, stype); + break; + } + case CFG_DTYPE_LOCALE: { + code = doSetConf(pItem, value, stype); + break; + } + case CFG_DTYPE_NONE: default: - TAOS_RETURN(TSDB_CODE_INVALID_CFG); + code = TSDB_CODE_INVALID_CFG; + break; } TAOS_RETURN(TSDB_CODE_SUCCESS); @@ -450,7 +470,7 @@ void cfgLock(SConfig *pCfg) { void cfgUnLock(SConfig *pCfg) { (void)taosThreadMutexUnlock(&pCfg->lock); } -int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer) { +int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer, bool isUpdateAll) { ECfgDynType dynType = isServer ? CFG_DYN_SERVER : CFG_DYN_CLIENT; cfgLock(pCfg); @@ -461,6 +481,11 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p cfgUnLock(pCfg); TAOS_RETURN(TSDB_CODE_INVALID_CFG); } + if (!isUpdateAll && (pItem->category & CFG_CATEGORY_GLOBAL) == 0) { + uError("failed to config:%s, not support update global config on only one dnode", name); + cfgUnLock(pCfg); + TAOS_RETURN(TSDB_CODE_INVALID_CFG); + } switch (pItem->dtype) { case CFG_DTYPE_STRING: { From a579abd0f06b966d3f3efc3093c0cc356f27790a Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Fri, 22 Nov 2024 17:52:17 +0800 Subject: [PATCH 26/99] Add dyn lazy type. --- include/util/tconfig.h | 20 +++++++--- source/common/src/tglobal.c | 78 ++++++++++++++++++------------------- source/util/src/tconfig.c | 27 ++++++++++--- 3 files changed, 74 insertions(+), 51 deletions(-) diff --git a/include/util/tconfig.h b/include/util/tconfig.h index cfccbc16c8..008e44f8ea 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -63,18 +63,26 @@ typedef enum { CFG_DYN_NONE = 0, CFG_DYN_SERVER = 1, CFG_DYN_CLIENT = 2, - CFG_DYN_BOTH = 3, - CFG_DYN_READONLY = 4, + CFG_DYN_SERVER_LAZY = 3, + CFG_DYN_CLIENT_LAZY = 4, + CFG_DYN_BOTH_LAZY = 5, + CFG_DYN_BOTH = 6, #ifdef TD_ENTERPRISE CFG_DYN_ENT_SERVER = CFG_DYN_SERVER, CFG_DYN_ENT_CLIENT = CFG_DYN_CLIENT, + CFG_DYN_ENT_READONLY = CFG_MODIFICATION_READONLY, + CFG_DYN_ENT_SERVER_LAZY = CFG_DYN_SERVER_LAZY, + CFG_DYN_ENT_CLIENT_LAZY = CFG_DYN_CLIENT_LAZY, + CFG_DYN_ENT_BOTH_LAZY = CFG_DYN_BOTH_LAZY, CFG_DYN_ENT_BOTH = CFG_DYN_BOTH, - CFG_DYN_ENT_READONLY = CFG_DYN_READONLY, + #else CFG_DYN_ENT_SERVER = CFG_DYN_NONE, CFG_DYN_ENT_CLIENT = CFG_DYN_NONE, + CFG_DYN_ENT_SERVER_LAZY = CFG_DYN_NONE, + CFG_DYN_ENT_CLIENT_LAZY = CFG_DYN_NONE, + CFG_DYN_ENT_BOTH_LAZY = CFG_DYN_NONE, CFG_DYN_ENT_BOTH = CFG_DYN_NONE, - CFG_DYN_ENT_READONLY = CFG_DYN_NONE, #endif } ECfgDynType; @@ -118,8 +126,8 @@ void cfgCleanup(SConfig *pCfg); int32_t cfgGetSize(SConfig *pCfg); SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName); int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype, bool lock); -int32_t cfgGetAndSetItem(SConfig *pCfg, SConfigItem **ppItem, const char *name, const char *value, ECfgSrcType stype, - bool lock); +int32_t cfgGetAndSetItem(SConfig *pCfg, SConfigItem **ppItem, const char *name, const char *value, ECfgSrcType stype, + bool lock); int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer, bool isUpdateAll); int32_t cfgCreateIter(SConfig *pConf, SConfigIter **ppIter); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index a45d3b59dc..121bbfc496 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -530,7 +530,7 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input int32_t taosAddClientLogCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddDir(pCfg, "configDir", configDir, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddDir(pCfg, "scriptDir", configDir, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddDir(pCfg, "logDir", tsLogDir, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddDir(pCfg, "logDir", tsLogDir, CFG_SCOPE_BOTH, CFG_DYN_BOTH_LAZY, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN( cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfLogLines", tsNumOfLogLines, 1000, 2000000000, CFG_SCOPE_BOTH, @@ -613,8 +613,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { cfgAddFloat(pCfg, "minimalTmpDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, CFG_SCOPE_BOTH, CFG_DYN_BOTH, - CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, CFG_SCOPE_BOTH, + CFG_DYN_BOTH_LAZY, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN( cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, CFG_SCOPE_CLIENT, CFG_DYN_ENT_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "queryTableNotExistAsEmpty", tsQueryTbNotExistAsEmpty, CFG_SCOPE_CLIENT, @@ -643,12 +643,12 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { cfgAddString(pCfg, "smlTsDefaultName", tsSmlTsDefaultName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN( cfgAddBool(pCfg, "smlDot2Underline", tsSmlDot2Underline, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, CFG_SCOPE_CLIENT, CFG_DYN_BOTH, - CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, CFG_SCOPE_CLIENT, + CFG_DYN_BOTH_LAZY, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxInsertBatchRows", tsMaxInsertBatchRows, 1, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL) != 0); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_BOTH, CFG_DYN_BOTH, - CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_BOTH, + CFG_DYN_BOTH_LAZY, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "useAdapter", tsUseAdapter, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN( cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); @@ -674,8 +674,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { CFG_CATEGORY_GLOBAL)); tsShareConnLimit = TRANGE(tsShareConnLimit, 1, 512); - TAOS_CHECK_RETURN( - cfgAddInt32(pCfg, "shareConnLimit", tsShareConnLimit, 1, 512, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "shareConnLimit", tsShareConnLimit, 1, 512, CFG_SCOPE_BOTH, CFG_DYN_SERVER_LAZY, + CFG_CATEGORY_GLOBAL)); tsReadTimeout = TRANGE(tsReadTimeout, 64, 24 * 3600 * 7); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "readTimeout", tsReadTimeout, 64, 24 * 3600 * 7, CFG_SCOPE_BOTH, CFG_DYN_NONE, @@ -686,16 +686,16 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); tsKeepAliveIdle = TRANGE(tsKeepAliveIdle, 1, 72000); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "keepAliveIdle", tsKeepAliveIdle, 1, 7200000, CFG_SCOPE_BOTH, CFG_DYN_BOTH, + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "keepAliveIdle", tsKeepAliveIdle, 1, 7200000, CFG_SCOPE_BOTH, CFG_DYN_BOTH_LAZY, CFG_CATEGORY_GLOBAL)); tsNumOfTaskQueueThreads = tsNumOfCores * 2; tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 16); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT, - CFG_DYN_CLIENT, CFG_CATEGORY_GLOBAL)); + CFG_DYN_CLIENT_LAZY, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN( - cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); + cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH_LAZY, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "multiResultFunctionStarReturnTags", tsMultiResultFunctionStarReturnTags, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); @@ -717,7 +717,7 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) { TAOS_CHECK_RETURN( cfgAddBool(pCfg, "enableCoreFile", tsEnableCoreFile, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN( - cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_LOCAL)); + cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_BOTH_LAZY, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "simdEnable", tsSIMDEnable, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "AVX512Enable", tsAVX512Enable, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); @@ -794,21 +794,21 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "retentionSpeedLimitMB", tsRetentionSpeedLimitMB, 0, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 4, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 4, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfVnodeRsmaThreads", tsNumOfVnodeRsmaThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfVnodeRsmaThreads", tsNumOfVnodeRsmaThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); @@ -835,7 +835,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddString(pCfg, "slowLogScope", tsSlowLogScopeString, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "slowLogExceptDb", tsSlowLogExceptDb, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); @@ -852,9 +852,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_BOTH, CFG_DYN_BOTH,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, CFG_SCOPE_BOTH, CFG_DYN_NONE,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "rsyncPort", tsRsyncPort, 1, 65535, CFG_SCOPE_BOTH, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "snodeAddress", tsSnodeAddress, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "rsyncPort", tsRsyncPort, 1, 65535, CFG_SCOPE_BOTH, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "snodeAddress", tsSnodeAddress, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); @@ -879,37 +879,37 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "udf", tsStartUdfd, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "udfdLdLibPath", tsUdfdLdLibPath, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "udf", tsStartUdfd, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "udfdLdLibPath", tsUdfdLdLibPath, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "streamAggCnt", tsStreamAggCnt, 2, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1800, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "concurrentCheckpoint", tsMaxConcurrentCheckpoint, 1, 10, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "curRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "ifAdtFse", tsIfAdtFse, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "compressor", tsCompressor, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "resolveFQDNRetryTime", tsResolveFQDNRetryTime, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey[0], CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint[0], CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey[0], CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint[0], CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); // min free disk space used to check if the disk is full [50MB, 1GB] diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 72dcd8f284..995396b49d 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -470,23 +470,38 @@ void cfgLock(SConfig *pCfg) { void cfgUnLock(SConfig *pCfg) { (void)taosThreadMutexUnlock(&pCfg->lock); } -int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer, bool isUpdateAll) { - ECfgDynType dynType = isServer ? CFG_DYN_SERVER : CFG_DYN_CLIENT; +int32_t checkItemDyn(SConfigItem *pItem, bool isServer) { + if (pItem->dynScope == CFG_DYN_NONE) { + return TSDB_CODE_SUCCESS; + } + if (isServer) { + if (pItem->dynScope == CFG_DYN_ENT_CLIENT || pItem->dynScope == CFG_DYN_ENT_CLIENT_LAZY) { + return TSDB_CODE_INVALID_CFG; + } + } else { + if (pItem->dynScope == CFG_DYN_ENT_SERVER || pItem->dynScope == CFG_DYN_ENT_SERVER_LAZY) { + return TSDB_CODE_INVALID_CFG; + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer, bool isUpdateAll) { cfgLock(pCfg); SConfigItem *pItem = cfgGetItem(pCfg, name); - if (!pItem || (pItem->dynScope & dynType) == 0) { - uError("failed to config:%s, not support update this config", name); + if (pItem == NULL) { cfgUnLock(pCfg); - TAOS_RETURN(TSDB_CODE_INVALID_CFG); + TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); } + TAOS_CHECK_RETURN(checkItemDyn(pItem, isServer)); if (!isUpdateAll && (pItem->category & CFG_CATEGORY_GLOBAL) == 0) { uError("failed to config:%s, not support update global config on only one dnode", name); cfgUnLock(pCfg); TAOS_RETURN(TSDB_CODE_INVALID_CFG); } - switch (pItem->dtype) { case CFG_DTYPE_STRING: { if (strcasecmp(name, "slowLogScope") == 0) { From 8ffa8811f5c3bd225bdd70603d1f54612d50a93a Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 25 Nov 2024 10:24:44 +0800 Subject: [PATCH 27/99] Rename persistLocalConfig to taosPersistLocalConfig. --- include/common/tglobal.h | 4 +-- source/common/src/tglobal.c | 4 +-- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 32 +++++++++++++++++---- source/util/src/tconfig.c | 1 - 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 5ebfd24d5e..e49f768f55 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -296,8 +296,8 @@ void taosLocalCfgForbiddenToChange(char *name, bool *forbidden); int8_t taosGranted(int8_t type); int32_t taosSetSlowLogScope(char *pScopeStr, int32_t *pScope); -int32_t persistGlobalConfig(SArray *array, const char *path, int32_t version); -int32_t persistLocalConfig(const char *path); +int32_t taosPersistGlobalConfig(SArray *array, const char *path, int32_t version); +int32_t taosPersistLocalConfig(const char *path); int32_t localConfigSerialize(SArray *array, char **serialized); int32_t tSerializeSConfigArray(SEncoder *pEncoder, SArray *array); int32_t tDeserializeSConfigArray(SDecoder *pDecoder, SArray *array); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 121bbfc496..98ebc3aeff 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -2800,7 +2800,7 @@ int32_t localConfigSerialize(SArray *array, char **serialized) { } // TODO:close file when error -int32_t persistGlobalConfig(SArray *array, const char *path, int32_t version) { +int32_t taosPersistGlobalConfig(SArray *array, const char *path, int32_t version) { // TODO: just tmp ,refactor later int32_t code = 0; char *buffer = NULL; @@ -2836,7 +2836,7 @@ int32_t persistGlobalConfig(SArray *array, const char *path, int32_t version) { return TSDB_CODE_SUCCESS; } -int32_t persistLocalConfig(const char *path) { +int32_t taosPersistLocalConfig(const char *path) { // TODO: just tmp ,refactor later int32_t code = 0; char *buffer = NULL; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 105f597d22..7021e45173 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -20,6 +20,7 @@ #include "systable.h" #include "tanal.h" #include "tchecksum.h" +#include "tutil.h" extern SConfig *tsCfg; @@ -285,6 +286,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { static void dmProcessConfigRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { const STraceId *trace = &pRsp->info.traceId; + int32_t code = 0; SConfigRsp configRsp = {0}; if (pRsp->code != 0) { @@ -305,7 +307,12 @@ static void dmProcessConfigRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { // Try to use cfg file in current dnode. if (configRsp.forceReadConfig) { if (configRsp.isConifgVerified) { - persistGlobalConfig(taosGetGlobalCfg(tsCfg), pMgmt->path, configRsp.cver); + code = taosPersistGlobalConfig(taosGetGlobalCfg(tsCfg), pMgmt->path, configRsp.cver); + if (code != TSDB_CODE_SUCCESS) { + dError("failed to persist global config since %s", tstrerror(code)); + goto _exit; + } + needUpdate = true; } else { // log the difference configurations printConfigNotMatch(configRsp.array); @@ -315,13 +322,20 @@ static void dmProcessConfigRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { // Try to use cfg from mnode sdb. if (!configRsp.isVersionVerified) { needUpdate = true; - persistGlobalConfig(configRsp.array, pMgmt->path, configRsp.cver); + code = taosPersistGlobalConfig(configRsp.array, pMgmt->path, configRsp.cver); + if (code != TSDB_CODE_SUCCESS) { + dError("failed to persist global config since %s", tstrerror(code)); + goto _exit; + } } } if (needUpdate) { setAllConfigs(tsCfg); } - persistLocalConfig(pMgmt->path); + code = taosPersistLocalConfig(pMgmt->path); + if (code != TSDB_CODE_SUCCESS) { + dError("failed to persist local config since %s", tstrerror(code)); + } tsConfigInited = 1; } _exit: @@ -453,6 +467,7 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { SConfig *pCfg = taosGetCfg(); SConfigItem *pItem = NULL; + code = cfgGetAndSetItem(pCfg, &pItem, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_CMD, true); if (code != 0) { if (strncasecmp(cfgReq.config, "resetlog", strlen("resetlog")) == 0) { @@ -466,11 +481,16 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { tsdmConfigVersion = cfgReq.version; } if (pItem->category == CFG_CATEGORY_GLOBAL) { - persistGlobalConfig(taosGetGlobalCfg(pCfg), pMgmt->path, tsdmConfigVersion); + code = taosPersistGlobalConfig(taosGetGlobalCfg(pCfg), pMgmt->path, tsdmConfigVersion); + if (code != TSDB_CODE_SUCCESS) { + dError("failed to persist global config since %s", tstrerror(code)); + } } else { - persistLocalConfig(pMgmt->path); + code = taosPersistLocalConfig(pMgmt->path); + if (code != TSDB_CODE_SUCCESS) { + dError("failed to persist local config since %s", tstrerror(code)); + } } - return TSDB_CODE_SUCCESS; } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 995396b49d..213a41c1a3 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -377,7 +377,6 @@ int32_t cfgGetAndSetItem(SConfig *pCfg, SConfigItem **pItem, const char *name, c (void)taosThreadMutexUnlock(&pCfg->lock); TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); } - TAOS_CHECK_RETURN(cfgSetItemVal(*pItem, name, value, stype)); if (lock) { From a63c132fbaffd4f402ba1a3a8c704311776e4b44 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 25 Nov 2024 10:37:12 +0800 Subject: [PATCH 28/99] Add config item alter lazy mode. --- include/common/tglobal.h | 1 + source/common/src/tglobal.c | 8 ++++++++ source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 10 +++++++--- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index e49f768f55..c95a1ad662 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -305,6 +305,7 @@ int32_t setAllConfigs(SConfig *pCfg); void printConfigNotMatch(SArray *array); int32_t compareSConfigItemArrays(SArray *mArray, const SArray *dArray, SArray *diffArray); +bool isConifgItemLazyMode(SConfigItem *item); #ifdef __cplusplus } #endif diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 98ebc3aeff..54e948b4b4 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -3036,4 +3036,12 @@ void printConfigNotMatch(SArray *array) { } } } +} + +bool isConifgItemLazyMode(SConfigItem *item) { + if (item->dynScope == CFG_DYN_CLIENT_LAZY || item->dynScope == CFG_DYN_SERVER_LAZY || + item->dynScope == CFG_DYN_BOTH_LAZY) { + return true; + } + return false; } \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 7021e45173..8eda3c807a 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -476,10 +476,11 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { return code; } } - TAOS_CHECK_RETURN(taosCfgDynamicOptions(pCfg, cfgReq.config, true)); - if (cfgReq.version > 0) { - tsdmConfigVersion = cfgReq.version; + if (isConifgItemLazyMode(pItem)) { + TAOS_CHECK_RETURN(taosCfgDynamicOptions(pCfg, cfgReq.config, true)); + return TSDB_CODE_INVALID_MSG; } + if (pItem->category == CFG_CATEGORY_GLOBAL) { code = taosPersistGlobalConfig(taosGetGlobalCfg(pCfg), pMgmt->path, tsdmConfigVersion); if (code != TSDB_CODE_SUCCESS) { @@ -491,6 +492,9 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { dError("failed to persist local config since %s", tstrerror(code)); } } + if (cfgReq.version > 0) { + tsdmConfigVersion = cfgReq.version; + } return TSDB_CODE_SUCCESS; } From 9ebff57e09e78e5034d9bdc21833e2ccb10b6a1f Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 25 Nov 2024 14:48:01 +0800 Subject: [PATCH 29/99] Fix merge error. --- include/common/tglobal.h | 1 - source/common/src/tglobal.c | 9 --------- 2 files changed, 10 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 07fe1450da..67bec995d2 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -82,7 +82,6 @@ extern int32_t tsNumOfRpcSessions; extern int32_t tsShareConnLimit; extern int32_t tsReadTimeout; extern int32_t tsTimeToGetAvailableConn; -extern int32_t tsKeepAliveIdle; extern int32_t tsNumOfCommitThreads; extern int32_t tsNumOfTaskQueueThreads; extern int32_t tsNumOfMnodeQueryThreads; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 0ebc84b1cd..2272aa7df1 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -528,7 +528,6 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input } int32_t taosAddClientLogCfg(SConfig *pCfg) { -<<<<<<< HEAD TAOS_CHECK_RETURN(cfgAddDir(pCfg, "configDir", configDir, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddDir(pCfg, "scriptDir", configDir, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddDir(pCfg, "logDir", tsLogDir, CFG_SCOPE_BOTH, CFG_DYN_BOTH_LAZY, CFG_CATEGORY_LOCAL)); @@ -659,7 +658,6 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { CFG_DYN_BOTH_LAZY, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "useAdapter", tsUseAdapter, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN( -<<<<<<< HEAD cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, CFG_SCOPE_CLIENT, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); @@ -859,11 +857,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddString(pCfg, "snodeAddress", tsSnodeAddress, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "telemetryInterval", tsTelemInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_SERVER, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tmqRowSize", tmqRowSize, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); @@ -2254,7 +2247,6 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"queryRsmaTolerance", &tsQueryRsmaTolerance}, {"countAlwaysReturnValue", &tsCountAlwaysReturnValue}, {"uptimeInterval", &tsUptimeInterval}, - {"slowLogThresholdTest", &tsSlowLogThresholdTest}, {"slowLogMaxLen", &tsSlowLogMaxLen}, {"slowLogScope", &tsSlowLogScope}, @@ -2288,7 +2280,6 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"trimVDbIntervalSec", &tsTrimVDbIntervalSec}, {"ttlChangeOnWrite", &tsTtlChangeOnWrite}, - {"keepAliveIdle", &tsKeepAliveIdle}, {"logKeepDays", &tsLogKeepDays}, {"maxStreamBackendCache", &tsMaxStreamBackendCache}, {"mqRebalanceInterval", &tsMqRebalanceInterval}, From 5d2bc35569473a921e6774084597a216fbd91816 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 25 Nov 2024 16:23:34 +0800 Subject: [PATCH 30/99] Add more log info. --- include/dnode/mnode/mnode.h | 6 - include/util/tconfig.h | 7 - source/common/src/tglobal.c | 138 +++++++++++--------- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 5 +- source/dnode/mnode/impl/src/mndConfig.c | 24 ++-- 5 files changed, 94 insertions(+), 86 deletions(-) diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 9bdb6eac37..780cdbf34e 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -41,12 +41,6 @@ typedef struct { int64_t lastIndex; } SMnodeOpt; -typedef struct { - int32_t version; - SArray *pArray; - TdThreadMutex mutex; -} SMnodeConfig; - /* ------------------------ SMnode ------------------------ */ /** * @brief Open a mnode. diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 008e44f8ea..71d871a775 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -53,11 +53,6 @@ typedef enum { typedef enum { CFG_SCOPE_SERVER, CFG_SCOPE_CLIENT, CFG_SCOPE_BOTH } ECfgScopeType; typedef enum { CFG_CATEGORY_GLOBAL, CFG_CATEGORY_LOCAL } ECfgCategoryType; -typedef enum { - CFG_DYNAMIC_MODIFICATION_SUPPORT, - CFG_DYNAMIC_MODIFICATION_NOT_SUPPORT, - CFG_MODIFICATION_READONLY -} ECfgModificationType; typedef enum { CFG_DYN_NONE = 0, @@ -70,12 +65,10 @@ typedef enum { #ifdef TD_ENTERPRISE CFG_DYN_ENT_SERVER = CFG_DYN_SERVER, CFG_DYN_ENT_CLIENT = CFG_DYN_CLIENT, - CFG_DYN_ENT_READONLY = CFG_MODIFICATION_READONLY, CFG_DYN_ENT_SERVER_LAZY = CFG_DYN_SERVER_LAZY, CFG_DYN_ENT_CLIENT_LAZY = CFG_DYN_CLIENT_LAZY, CFG_DYN_ENT_BOTH_LAZY = CFG_DYN_BOTH_LAZY, CFG_DYN_ENT_BOTH = CFG_DYN_BOTH, - #else CFG_DYN_ENT_SERVER = CFG_DYN_NONE, CFG_DYN_ENT_CLIENT = CFG_DYN_NONE, diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 2272aa7df1..44f4c10f49 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1927,6 +1927,7 @@ int32_t cfgDeserialize(SArray *array, char *buf, bool isGlobal) { int32_t sz = taosArrayGetSize(array); cJSON *configs = cJSON_GetObjectItem(pRoot, "configs"); if (configs == NULL) { + uError("failed to get configs from json, since %s", cJSON_GetErrorPtr()); cJSON_Delete(pRoot); return TSDB_CODE_OUT_OF_MEMORY; } @@ -1963,7 +1964,6 @@ int32_t cfgDeserialize(SArray *array, char *buf, bool isGlobal) { } } } - cJSON_Delete(pRoot); return TSDB_CODE_SUCCESS; } @@ -1978,6 +1978,7 @@ int32_t readCfgFile(const char *path, bool isGlobal) { array = taosGetLocalCfg(tsCfg); snprintf(filename, sizeof(filename), "%s%sconfig%slocal.json", path, TD_DIRSEP, TD_DIRSEP); } + uInfo("start to read config file:%s", filename); int64_t fileSize = 0; char *buf = NULL; @@ -1985,27 +1986,33 @@ int32_t readCfgFile(const char *path, bool isGlobal) { if (terrno != ENOENT) { uError("failed to stat file:%s , since %s", filename, tstrerror(code)); code = terrno; + goto _exit; } - TAOS_RETURN(TSDB_CODE_SUCCESS); } TdFilePtr pFile = taosOpenFile(filename, TD_FILE_READ); if (pFile == NULL) { + uError("failed to open file:%s , since %s", filename, tstrerror(code)); code = terrno; - TAOS_RETURN(code); + goto _exit; } buf = (char *)taosMemoryMalloc(fileSize + 1); if (taosReadFile(pFile, buf, fileSize) != fileSize) { uError("failed to read file:%s , config since %s", filename, tstrerror(code)); - (void)taosCloseFile(&pFile); - taosMemoryFree(buf); - TAOS_RETURN(terrno); + code = terrno; + goto _exit; } - char *serialized = NULL; code = cfgDeserialize(array, buf, isGlobal); if (code != TSDB_CODE_SUCCESS) { uError("failed to deserialize config from %s since %s", filename, tstrerror(code)); - TAOS_RETURN(code); + goto _exit; } + +_exit: + if (code != TSDB_CODE_SUCCESS) { + uError("failed to read config from %s since %s", filename, tstrerror(code)); + } + taosMemoryFree(buf); + (void)taosCloseFile(&pFile); TAOS_RETURN(code); } @@ -2015,6 +2022,7 @@ int32_t tryLoadCfgFromDataDir(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(tsCfg, pItem, "forceReadConfig"); tsForceReadConfig = pItem->i32; if (!tsForceReadConfig) { + uInfo("load config from tsDataDir:%s", tsDataDir); code = readCfgFile(tsDataDir, false); if (code != TSDB_CODE_SUCCESS) { uError("failed to read local config from %s since %s", tsDataDir, tstrerror(code)); @@ -2695,17 +2703,13 @@ int8_t taosGranted(int8_t type) { int32_t globalConfigSerialize(int32_t version, SArray *array, char **serialized) { char buf[30]; cJSON *json = cJSON_CreateObject(); - if (json == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - cJSON_AddNumberToObject(json, "version", version); + if (json == NULL) goto _exit; + json = cJSON_AddNumberToObject(json, "version", version); + if (json == NULL) goto _exit; int sz = taosArrayGetSize(array); cJSON *cField = cJSON_CreateObject(); - if (cField == NULL) { - cJSON_Delete(json); - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); - } + if (cField == NULL) goto _exit; // cjson only support int32_t or double // string are used to prohibit the loss of precision @@ -2716,50 +2720,56 @@ int32_t globalConfigSerialize(int32_t version, SArray *array, char **serialized) case CFG_DTYPE_NONE: break; case CFG_DTYPE_BOOL: - cJSON_AddBoolToObject(cField, item->name, item->bval); + cField = cJSON_AddBoolToObject(cField, item->name, item->bval); + if (cField == NULL) goto _exit; break; case CFG_DTYPE_INT32: - cJSON_AddNumberToObject(cField, item->name, item->i32); + cField = cJSON_AddNumberToObject(cField, item->name, item->i32); + if (cField == NULL) goto _exit; break; case CFG_DTYPE_INT64: (void)sprintf(buf, "%" PRId64, item->i64); - cJSON_AddStringToObject(cField, item->name, buf); + cField = cJSON_AddStringToObject(cField, item->name, buf); + if (cField == NULL) goto _exit; break; case CFG_DTYPE_FLOAT: case CFG_DTYPE_DOUBLE: (void)sprintf(buf, "%f", item->fval); - cJSON_AddStringToObject(cField, item->name, buf); + cField = cJSON_AddStringToObject(cField, item->name, buf); + if (cField == NULL) goto _exit; break; case CFG_DTYPE_STRING: case CFG_DTYPE_DIR: case CFG_DTYPE_LOCALE: case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: - cJSON_AddStringToObject(cField, item->name, item->str); + cField = cJSON_AddStringToObject(cField, item->name, item->str); + if (cField == NULL) goto _exit; break; } } } - cJSON_AddItemToObject(json, "configs", cField); + if (!cJSON_AddItemToObject(json, "configs", cField)) { + goto _exit; + } *serialized = cJSON_Print(json); +_exit: + if (terrno != TSDB_CODE_SUCCESS) { + uError("failed to serialize global config since %s", tstrerror(terrno)); + } cJSON_Delete(json); - return TSDB_CODE_SUCCESS; + return terrno; } int32_t localConfigSerialize(SArray *array, char **serialized) { char buf[30]; cJSON *json = cJSON_CreateObject(); - if (json == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } + if (json == NULL) goto _exit; int sz = taosArrayGetSize(array); cJSON *cField = cJSON_CreateObject(); - if (array == NULL) { - cJSON_Delete(json); - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); - } + if (cField == NULL) goto _exit; // cjson only support int32_t or double // string are used to prohibit the loss of precision for (int i = 0; i < sz; i++) { @@ -2769,40 +2779,48 @@ int32_t localConfigSerialize(SArray *array, char **serialized) { case CFG_DTYPE_NONE: break; case CFG_DTYPE_BOOL: - cJSON_AddBoolToObject(cField, item->name, item->bval); + cField = cJSON_AddBoolToObject(cField, item->name, item->bval); + if (cField == NULL) goto _exit; break; case CFG_DTYPE_INT32: - cJSON_AddNumberToObject(cField, item->name, item->i32); + cField = cJSON_AddNumberToObject(cField, item->name, item->i32); + if (cField == NULL) goto _exit; break; case CFG_DTYPE_INT64: (void)sprintf(buf, "%" PRId64, item->i64); - cJSON_AddStringToObject(cField, item->name, buf); + cField = cJSON_AddStringToObject(cField, item->name, buf); + if (cField == NULL) goto _exit; break; case CFG_DTYPE_FLOAT: case CFG_DTYPE_DOUBLE: (void)sprintf(buf, "%f", item->fval); - cJSON_AddStringToObject(cField, item->name, buf); + cField = cJSON_AddStringToObject(cField, item->name, buf); + if (cField == NULL) goto _exit; break; case CFG_DTYPE_STRING: case CFG_DTYPE_DIR: case CFG_DTYPE_LOCALE: case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: - cJSON_AddStringToObject(cField, item->name, item->str); + cField = cJSON_AddStringToObject(cField, item->name, item->str); + if (cField == NULL) goto _exit; break; } } } cJSON_AddItemToObject(json, "configs", cField); *serialized = cJSON_Print(json); +_exit: + if (terrno != TSDB_CODE_SUCCESS) { + uError("failed to serialize local config since %s", tstrerror(terrno)); + } cJSON_Delete(json); - return TSDB_CODE_SUCCESS; + return terrno; } -// TODO:close file when error int32_t taosPersistGlobalConfig(SArray *array, const char *path, int32_t version) { - // TODO: just tmp ,refactor later int32_t code = 0; + int32_t lino = 0; char *buffer = NULL; TdFilePtr pFile = NULL; char filepath[CONFIG_FILE_LEN] = {0}; @@ -2810,12 +2828,7 @@ int32_t taosPersistGlobalConfig(SArray *array, const char *path, int32_t version snprintf(filepath, sizeof(filepath), "%s%sconfig", path, TD_DIRSEP); snprintf(filename, sizeof(filename), "%s%sconfig%sglobal.json", path, TD_DIRSEP, TD_DIRSEP); - // TODO(beryl) need to check if the file is existed - if (taosMkDir(filepath) != 0) { - code = TAOS_SYSTEM_ERROR(errno); - uError("failed to create dir:%s since %s", filepath, tstrerror(code)); - TAOS_RETURN(code); - } + TAOS_CHECK_GOTO(taosMkDir(filepath), &lino, _exit); TdFilePtr pConfigFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); @@ -2826,19 +2839,21 @@ int32_t taosPersistGlobalConfig(SArray *array, const char *path, int32_t version TAOS_RETURN(code); } char *serialized = NULL; - code = globalConfigSerialize(version, array, &serialized); + TAOS_CHECK_GOTO(globalConfigSerialize(version, array, &serialized), &lino, _exit); + + TAOS_CHECK_GOTO(taosWriteFile(pConfigFile, serialized, strlen(serialized)), &lino, _exit); + +_exit: if (code != TSDB_CODE_SUCCESS) { - uError("failed to serialize local config since %s", tstrerror(code)); - TAOS_RETURN(code); + uError("failed to persist global config at line:%d, since %s", lino, tstrerror(code)); } - taosWriteFile(pConfigFile, serialized, strlen(serialized)); (void)taosCloseFile(&pConfigFile); - return TSDB_CODE_SUCCESS; + return code; } int32_t taosPersistLocalConfig(const char *path) { - // TODO: just tmp ,refactor later int32_t code = 0; + int32_t lino = 0; char *buffer = NULL; TdFilePtr pFile = NULL; char filepath[CONFIG_FILE_LEN] = {0}; @@ -2847,11 +2862,7 @@ int32_t taosPersistLocalConfig(const char *path) { snprintf(filename, sizeof(filename), "%s%sconfig%slocal.json", path, TD_DIRSEP, TD_DIRSEP); // TODO(beryl) need to check if the file is existed - if (taosMkDir(filepath) != 0) { - code = TAOS_SYSTEM_ERROR(errno); - uError("failed to create dir:%s since %s", filepath, tstrerror(code)); - TAOS_RETURN(code); - } + TAOS_CHECK_GOTO(taosMkDir(filepath), &lino, _exit); TdFilePtr pConfigFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); @@ -2861,15 +2872,18 @@ int32_t taosPersistLocalConfig(const char *path) { uError("failed to open file:%s since %s", filename, tstrerror(code)); TAOS_RETURN(code); } + char *serialized = NULL; - code = localConfigSerialize(taosGetLocalCfg(tsCfg), &serialized); - if (code != TSDB_CODE_SUCCESS) { - uError("failed to serialize local config since %s", tstrerror(code)); - TAOS_RETURN(code); - } - taosWriteFile(pConfigFile, serialized, strlen(serialized)); + TAOS_CHECK_GOTO(localConfigSerialize(taosGetLocalCfg(tsCfg), &serialized), &lino, _exit); + TAOS_CHECK_GOTO(taosWriteFile(pConfigFile, serialized, strlen(serialized)), &lino, _exit); + (void)taosCloseFile(&pConfigFile); - return TSDB_CODE_SUCCESS; +_exit: + if (code != TSDB_CODE_SUCCESS) { + uError("failed to persist global config at line:%d, since %s", lino, tstrerror(code)); + } + (void)taosCloseFile(&pConfigFile); + return code; } int32_t tSerializeSConfigArray(SEncoder *pEncoder, SArray *array) { diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index ccb81eb034..1467f5adfa 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -290,8 +290,7 @@ static void dmProcessConfigRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { if (pRsp->code != 0) { if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pMgmt->pData->dropped && pMgmt->pData->dnodeId > 0) { - dGInfo("dnode:%d, set to dropped since not exist in mnode, statusSeq:%d", pMgmt->pData->dnodeId, - pMgmt->statusSeq); + dGInfo("dnode:%d, set to dropped since not exist in mnode", pMgmt->pData->dnodeId); pMgmt->pData->dropped = 1; if (dmWriteEps(pMgmt->pData) != 0) { dError("failed to write dnode file"); @@ -306,6 +305,7 @@ static void dmProcessConfigRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { // Try to use cfg file in current dnode. if (configRsp.forceReadConfig) { if (configRsp.isConifgVerified) { + uInfo("force read config and check config verified"); code = taosPersistGlobalConfig(taosGetGlobalCfg(tsCfg), pMgmt->path, configRsp.cver); if (code != TSDB_CODE_SUCCESS) { dError("failed to persist global config since %s", tstrerror(code)); @@ -320,6 +320,7 @@ static void dmProcessConfigRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { } // Try to use cfg from mnode sdb. if (!configRsp.isVersionVerified) { + uInfo("config version not verified, update config"); needUpdate = true; code = taosPersistGlobalConfig(configRsp.array, pMgmt->path, configRsp.cver); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index a8880ce94c..72b403df6a 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -141,7 +141,7 @@ SSdbRow *mndCfgActionDecode(SSdbRaw *pRaw) { obj = sdbGetRowObj(pRow); if (obj == NULL) goto _OVER; int32_t dataPos = 0; - // TODO(beryl):free it. + SDB_GET_BINARY(pRaw, dataPos, obj->name, CFG_NAME_MAX_LEN, _OVER) SDB_GET_INT32(pRaw, dataPos, (int32_t *)&obj->dtype, _OVER) switch (obj->dtype) { @@ -161,6 +161,8 @@ SSdbRow *mndCfgActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &len, _OVER) char *buf = taosMemoryMalloc(len + 1); SDB_GET_BINARY(pRaw, dataPos, buf, len, _OVER) + obj->fval = atof(buf); + taosMemoryFree(buf); break; case CFG_DTYPE_STRING: case CFG_DTYPE_DIR: @@ -180,6 +182,7 @@ _OVER: if (terrno != 0) { mError("cfg failed to decode from raw:%p since %s", pRaw, terrstr()); taosMemoryFreeClear(pRow); + taosMemoryFreeClear(obj->str); return NULL; } @@ -223,7 +226,6 @@ static int32_t mndProcessConfigReq(SRpcMsg *pReq) { configRsp.array = diffArray; } else { configRsp.isConifgVerified = 1; - taosArrayDestroy(diffArray); } } else { configRsp.array = taosGetGlobalCfg(tsCfg); @@ -235,17 +237,21 @@ static int32_t mndProcessConfigReq(SRpcMsg *pReq) { } int32_t contLen = tSerializeSConfigRsp(NULL, 0, &configRsp); - void *pHead = rpcMallocCont(contLen); + if (contLen < 0) { + code = contLen; + goto _OVER; + } + void *pHead = rpcMallocCont(contLen); contLen = tSerializeSConfigRsp(pHead, contLen, &configRsp); - taosArrayDestroy(diffArray); if (contLen < 0) { code = contLen; goto _OVER; } pReq->info.rspLen = contLen; pReq->info.rsp = pHead; -_OVER: +_OVER: + taosArrayDestroy(diffArray); mndReleaseDnode(pMnode, pDnode); return TSDB_CODE_SUCCESS; } @@ -254,6 +260,7 @@ int32_t mndInitWriteCfg(SMnode *pMnode) { int code = -1; size_t sz = 0; + mInfo("init write cfg to sdb"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "init-write-config"); if (pTrans == NULL) { mError("failed to init write cfg in create trans, since %s", terrstr()); @@ -264,6 +271,7 @@ int32_t mndInitWriteCfg(SMnode *pMnode) { SConfigObj *obj = mndInitConfigVersion(); if ((code = mndSetCreateConfigCommitLogs(pTrans, obj)) != 0) { mError("failed to init mnd config version, since %s", terrstr()); + goto _OVER; } sz = taosArrayGetSize(taosGetGlobalCfg(tsCfg)); @@ -275,7 +283,6 @@ int32_t mndInitWriteCfg(SMnode *pMnode) { } taosMemoryFree(obj); } - if ((code = mndTransCheckConflict(pMnode, pTrans)) != 0) goto _OVER; if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER; _OVER: @@ -540,20 +547,19 @@ _err: TAOS_RETURN(code); } -// TODO(beryl):add more log. static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue) { int32_t code = -1; int32_t lino = -1; SConfigObj *pVersion = sdbAcquire(pMnode->pSdb, SDB_CFG, "tsmmConfigVersion"); if (pVersion == NULL) { - mWarn("failed to acquire tsmmConfigVersion while update config, since %s", terrstr()); + mError("failed to acquire tsmmConfigVersion while update config, since %s", terrstr()); code = terrno; goto _OVER; } pVersion->i32 = ++tsmmConfigVersion; SConfigObj *pObj = sdbAcquire(pMnode->pSdb, SDB_CFG, name); if (pObj == NULL) { - mWarn("failed to acquire mnd config:%s while update config, since %s", name, terrstr()); + mError("failed to acquire mnd config:%s while update config, since %s", name, terrstr()); code = terrno; goto _OVER; } From 1451cfb07690edbfb6488326c533490a03cb6b5c Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 25 Nov 2024 16:45:06 +0800 Subject: [PATCH 31/99] Fix ci file check error. --- source/common/src/tglobal.c | 19 ++++++++++++++----- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 6 +++++- source/dnode/mnode/impl/src/mndConfig.c | 11 +++++++++-- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 44f4c10f49..5419da17a8 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -2808,7 +2808,7 @@ int32_t localConfigSerialize(SArray *array, char **serialized) { } } } - cJSON_AddItemToObject(json, "configs", cField); + if (!cJSON_AddItemToObject(json, "configs", cField)) goto _exit; *serialized = cJSON_Print(json); _exit: if (terrno != TSDB_CODE_SUCCESS) { @@ -2890,7 +2890,7 @@ int32_t tSerializeSConfigArray(SEncoder *pEncoder, SArray *array) { int32_t code = 0; int32_t lino = 0; int32_t sz = taosArrayGetSize(array); - tEncodeI32(pEncoder, sz); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, sz)); for (int i = 0; i < sz; i++) { SConfigItem *item = (SConfigItem *)taosArrayGet(array, i); TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, item->name)); @@ -2930,7 +2930,7 @@ int32_t tDeserializeSConfigArray(SDecoder *pDecoder, SArray *array) { int32_t code = 0; int32_t lino = 0; int32_t sz = 0; - tDecodeI32(pDecoder, &sz); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &sz)); for (int i = 0; i < sz; i++) { SConfigItem item = {0}; TAOS_CHECK_EXIT(tDecodeCStr(pDecoder, &item.name)); @@ -2961,9 +2961,15 @@ int32_t tDeserializeSConfigArray(SDecoder *pDecoder, SArray *array) { break; } } - taosArrayPush(array, &item); + if (taosArrayPush(array, &item) == NULL) { + code = terrno; + goto _exit; + } } _exit: + if (code != TSDB_CODE_SUCCESS) { + uError("failed to deserialize SConfigItem at line:%d, since %s", lino, tstrerror(code)); + } return code; } @@ -3008,7 +3014,10 @@ int32_t compareSConfigItemArrays(SArray *mArray, const SArray *dArray, SArray *d SConfigItem *dItem = (SConfigItem *)taosArrayGet(dArray, i); if (!compareSConfigItem(mItem, dItem)) { code = TSDB_CODE_FAILED; - taosArrayPush(diffArray, mItem); + if (taosArrayPush(diffArray, mItem) == NULL) { + code = terrno; + return code; + } } } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 1467f5adfa..19e1c28486 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -330,7 +330,11 @@ static void dmProcessConfigRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { } } if (needUpdate) { - setAllConfigs(tsCfg); + code = setAllConfigs(tsCfg); + if (code != TSDB_CODE_SUCCESS) { + dError("failed to set all configs since %s", tstrerror(code)); + goto _exit; + } } code = taosPersistLocalConfig(pMgmt->path); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index 72b403df6a..eb2765c51b 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -215,7 +215,11 @@ static int32_t mndProcessConfigReq(SRpcMsg *pReq) { SDnodeObj *pDnode = NULL; int32_t code = -1; - tDeserializeSConfigReq(pReq->pCont, pReq->contLen, &configReq); + code = tDeserializeSConfigReq(pReq->pCont, pReq->contLen, &configReq); + if (code != 0) { + mError("failed to deserialize config req, since %s", terrstr()); + goto _OVER; + } SArray *diffArray = taosArrayInit(16, sizeof(SConfigItem)); SConfigRsp configRsp = {0}; configRsp.forceReadConfig = configReq.forceReadConfig; @@ -315,7 +319,10 @@ int32_t mndInitReadCfg(SMnode *pMnode) { mInfo("failed to acquire mnd config:%s, since %s", item->name, terrstr()); continue; } - cfgUpdateItem(item, newObj); + code = cfgUpdateItem(item, newObj); + if (code != 0) { + mError("failed to update mnd config:%s, since %s", item->name, terrstr()); + } sdbRelease(pMnode->pSdb, newObj); } _OVER: From c099791812f56722529ad1e97389208654200b02 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 25 Nov 2024 19:14:08 +0800 Subject: [PATCH 32/99] Fix can not start dnode. --- source/common/src/tglobal.c | 27 ++++++++++++++++----------- tests/pytest/util/log.py | 2 +- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 5419da17a8..40aff508f5 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -645,8 +645,6 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { cfgAddString(pCfg, "smlTsDefaultName", tsSmlTsDefaultName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN( cfgAddBool(pCfg, "smlDot2Underline", tsSmlDot2Underline, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, CFG_SCOPE_CLIENT, - CFG_DYN_BOTH_LAZY, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); @@ -794,8 +792,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddString(pCfg, "encryptScope", tsEncryptScope, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL)); @@ -1982,12 +1978,12 @@ int32_t readCfgFile(const char *path, bool isGlobal) { int64_t fileSize = 0; char *buf = NULL; - if (taosStatFile(filename, &fileSize, NULL, NULL) != 0) { + if (taosStatFile(filename, &fileSize, NULL, NULL) < 0) { if (terrno != ENOENT) { - uError("failed to stat file:%s , since %s", filename, tstrerror(code)); code = terrno; - goto _exit; + uTrace("failed to stat file:%s , since %s", filename, tstrerror(code)); } + TAOS_RETURN(TSDB_CODE_SUCCESS); } TdFilePtr pFile = taosOpenFile(filename, TD_FILE_READ); if (pFile == NULL) { @@ -2841,7 +2837,12 @@ int32_t taosPersistGlobalConfig(SArray *array, const char *path, int32_t version char *serialized = NULL; TAOS_CHECK_GOTO(globalConfigSerialize(version, array, &serialized), &lino, _exit); - TAOS_CHECK_GOTO(taosWriteFile(pConfigFile, serialized, strlen(serialized)), &lino, _exit); + if (taosWriteFile(pConfigFile, serialized, strlen(serialized) < 0)) { + lino = __LINE__; + code = TAOS_SYSTEM_ERROR(errno); + uError("failed to write file:%s since %s", filename, tstrerror(code)); + goto _exit; + } _exit: if (code != TSDB_CODE_SUCCESS) { @@ -2875,12 +2876,16 @@ int32_t taosPersistLocalConfig(const char *path) { char *serialized = NULL; TAOS_CHECK_GOTO(localConfigSerialize(taosGetLocalCfg(tsCfg), &serialized), &lino, _exit); - TAOS_CHECK_GOTO(taosWriteFile(pConfigFile, serialized, strlen(serialized)), &lino, _exit); + if (taosWriteFile(pConfigFile, serialized, strlen(serialized)) < 0) { + lino = __LINE__; + code = TAOS_SYSTEM_ERROR(errno); + uError("failed to write file:%s since %s", filename, tstrerror(code)); + goto _exit; + } - (void)taosCloseFile(&pConfigFile); _exit: if (code != TSDB_CODE_SUCCESS) { - uError("failed to persist global config at line:%d, since %s", lino, tstrerror(code)); + uError("failed to persist local config at line:%d, since %s", lino, tstrerror(code)); } (void)taosCloseFile(&pConfigFile); return code; diff --git a/tests/pytest/util/log.py b/tests/pytest/util/log.py index 000c907ea4..e6218faa12 100644 --- a/tests/pytest/util/log.py +++ b/tests/pytest/util/log.py @@ -15,7 +15,7 @@ import sys import os import time import datetime -from distutils.log import warn as printf +from setuptools import logging as printf class TDLog: From e4416d49359c51bf0f99dba53e1fb6199c6ee431 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 25 Nov 2024 19:34:48 +0800 Subject: [PATCH 33/99] Fix mem error. --- source/dnode/mnode/impl/src/mndConfig.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index eb2765c51b..978e3e33d7 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -31,6 +31,7 @@ enum CfgAlterType { }; static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pInMCfgReq, int32_t optLen, int32_t *pOutValue); +static int32_t taosGetConfigObjSize(SConfigObj *obj); static int32_t cfgUpdateItem(SConfigItem *pItem, SConfigObj *obj); static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue); static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq); @@ -60,13 +61,24 @@ int32_t mndInitConfig(SMnode *pMnode) { return sdbSetTable(pMnode->pSdb, table); } +int32_t taosGetConfigObjSize(SConfigObj *obj) { + int32_t size = sizeof(SConfigObj); + if (obj->dtype == CFG_DTYPE_STRING || obj->dtype == CFG_DTYPE_DIR || obj->dtype == CFG_DTYPE_LOCALE || + obj->dtype == CFG_DTYPE_CHARSET || obj->dtype == CFG_DTYPE_TIMEZONE) { + if (obj->str != NULL) { + size += strlen(obj->str); + } + } + return size; +} + SSdbRaw *mnCfgActionEncode(SConfigObj *obj) { int32_t code = 0; int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; char buf[30]; - int32_t size = sizeof(SConfigObj); + int32_t size = taosGetConfigObjSize(obj); SSdbRaw *pRaw = sdbAllocRaw(SDB_CFG, CFG_VER_NUMBER, size); if (pRaw == NULL) goto _OVER; From 2b3dca319a830b14805145878a36af36cd9a61a8 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 26 Nov 2024 10:07:00 +0800 Subject: [PATCH 34/99] Fix some ci error. --- source/dnode/mnode/impl/src/mndConfig.c | 2 +- tests/pytest/util/log.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index 978e3e33d7..563c14411d 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -66,7 +66,7 @@ int32_t taosGetConfigObjSize(SConfigObj *obj) { if (obj->dtype == CFG_DTYPE_STRING || obj->dtype == CFG_DTYPE_DIR || obj->dtype == CFG_DTYPE_LOCALE || obj->dtype == CFG_DTYPE_CHARSET || obj->dtype == CFG_DTYPE_TIMEZONE) { if (obj->str != NULL) { - size += strlen(obj->str); + size += strlen(obj->str) + 1; } } return size; diff --git a/tests/pytest/util/log.py b/tests/pytest/util/log.py index e6218faa12..000c907ea4 100644 --- a/tests/pytest/util/log.py +++ b/tests/pytest/util/log.py @@ -15,7 +15,7 @@ import sys import os import time import datetime -from setuptools import logging as printf +from distutils.log import warn as printf class TDLog: From 7ff032ad04d5a31931ce5a7a7019b6cdeea3ac2f Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 26 Nov 2024 10:41:11 +0800 Subject: [PATCH 35/99] Fix alter local var and alter str var. --- source/util/src/tconfig.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index e7361ac174..478d9516d3 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -496,7 +496,7 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); } TAOS_CHECK_RETURN(checkItemDyn(pItem, isServer)); - if (!isUpdateAll && (pItem->category & CFG_CATEGORY_GLOBAL) == 0) { + if ((!isUpdateAll) && (pItem->category == CFG_CATEGORY_GLOBAL)) { uError("failed to config:%s, not support update global config on only one dnode", name); cfgUnLock(pCfg); TAOS_RETURN(TSDB_CODE_INVALID_CFG); @@ -979,9 +979,10 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { for (size_t j = 0; j < sz; ++j) { SDiskCfg *pCfg = taosArrayGet(pItem->array, j); if (dump) { - (void)printf("%s %s %s l:%d p:%d d:%"PRIi8"\n", src, name, pCfg->dir, pCfg->level, pCfg->primary, pCfg->disable); + (void)printf("%s %s %s l:%d p:%d d:%" PRIi8 "\n", src, name, pCfg->dir, pCfg->level, pCfg->primary, + pCfg->disable); } else { - uInfo("%s %s %s l:%d p:%d d:%"PRIi8, src, name, pCfg->dir, pCfg->level, pCfg->primary, pCfg->disable); + uInfo("%s %s %s l:%d p:%d d:%" PRIi8, src, name, pCfg->dir, pCfg->level, pCfg->primary, pCfg->disable); } } break; From 594bcf10073e8596613f6d58d86a2143caf61cd4 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 26 Nov 2024 10:41:40 +0800 Subject: [PATCH 36/99] Fix alter local var and alter str var (add miss commit). --- source/common/src/tglobal.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 40aff508f5..01966c0536 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -2152,6 +2152,15 @@ static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize, uInfo("%s set from %f to %f", optName, *pVar, flag); *pVar = flag; } break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: { + char *pVar = pOptions[d].optionVar; + tstrncpy(pVar, pItem->str, strlen(pItem->str)); + uInfo("%s set to %s", optName, pVar); + } break; default: code = TSDB_CODE_INVALID_CFG; break; From d778ada8f983d579c469ba60d4f1ec93f1dfcb9e Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 26 Nov 2024 16:08:16 +0800 Subject: [PATCH 37/99] Add show --- include/common/tmsg.h | 3 +- include/libs/nodes/cmdnodes.h | 17 +-- include/util/tconfig.h | 1 + include/util/tdef.h | 56 +++------ source/client/inc/clientInt.h | 5 +- source/client/src/clientMsgHandler.c | 111 ++++++++++-------- source/common/src/msg/tmsg.c | 27 ++--- source/common/src/systable.c | 1 + source/common/src/tglobal.c | 38 +++--- source/common/src/tmisce.c | 26 +++-- source/dnode/mnode/impl/src/mndDnode.c | 153 ++++++++++++------------- source/libs/command/src/command.c | 8 +- source/libs/parser/src/parTranslater.c | 61 +++++----- source/util/src/tconfig.c | 26 +++++ 14 files changed, 278 insertions(+), 255 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 36bebc3d88..2ce738f2be 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -676,7 +676,7 @@ typedef struct { int32_t tsSlowLogThreshold; int32_t tsSlowLogMaxLen; int32_t tsSlowLogScope; - int32_t tsSlowLogThresholdTest; //Obsolete + int32_t tsSlowLogThresholdTest; // Obsolete char tsSlowLogExceptDb[TSDB_DB_NAME_LEN]; } SMonitorParas; @@ -2210,6 +2210,7 @@ typedef struct { char name[TSDB_CONFIG_OPTION_LEN + 1]; char value[TSDB_CONFIG_PATH_LEN + 1]; char scope[TSDB_CONFIG_SCOPE_LEN + 1]; + char category[TSDB_CONFIG_CATEGORY_LEN + 1]; char info[TSDB_CONFIG_INFO_LEN + 1]; } SVariablesInfo; diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 867f8c8efc..c59ca21b62 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -42,11 +42,12 @@ extern "C" { #define SHOW_CREATE_VIEW_RESULT_FIELD1_LEN (TSDB_VIEW_FNAME_LEN + 4 + VARSTR_HEADER_SIZE) #define SHOW_CREATE_VIEW_RESULT_FIELD2_LEN (TSDB_MAX_ALLOWED_SQL_LEN + VARSTR_HEADER_SIZE) -#define SHOW_LOCAL_VARIABLES_RESULT_COLS 4 +#define SHOW_LOCAL_VARIABLES_RESULT_COLS 5 #define SHOW_LOCAL_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE) #define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_PATH_LEN + VARSTR_HEADER_SIZE) #define SHOW_LOCAL_VARIABLES_RESULT_FIELD3_LEN (TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE) -#define SHOW_LOCAL_VARIABLES_RESULT_FIELD4_LEN (TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE) +#define SHOW_LOCAL_VARIABLES_RESULT_FIELD4_LEN (TSDB_CONFIG_CATEGORY_LEN + VARSTR_HEADER_SIZE) +#define SHOW_LOCAL_VARIABLES_RESULT_FIELD5_LEN (TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE) #define COMPACT_DB_RESULT_COLS 3 #define COMPACT_DB_RESULT_FIELD1_LEN 32 @@ -341,7 +342,7 @@ typedef struct SShowStmt { SNode* pTbName; // SValueNode EOperatorType tableCondType; EShowKind showKind; // show databases: user/system, show tables: normal/child, others NULL - bool withFull; // for show users full; + bool withFull; // for show users full; } SShowStmt; typedef struct SShowCreateDatabaseStmt { @@ -651,7 +652,7 @@ typedef struct SCreateTSMAStmt { bool ignoreExists; char tsmaName[TSDB_TABLE_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN]; - char tableName[TSDB_TABLE_NAME_LEN]; // base tb name or base tsma name + char tableName[TSDB_TABLE_NAME_LEN]; // base tb name or base tsma name char originalTbName[TSDB_TABLE_NAME_LEN]; STSMAOptions* pOptions; SNode* pPrevQuery; @@ -660,10 +661,10 @@ typedef struct SCreateTSMAStmt { } SCreateTSMAStmt; typedef struct SDropTSMAStmt { - ENodeType type; - bool ignoreNotExists; - char dbName[TSDB_DB_NAME_LEN]; - char tsmaName[TSDB_TABLE_NAME_LEN]; + ENodeType type; + bool ignoreNotExists; + char dbName[TSDB_DB_NAME_LEN]; + char tsmaName[TSDB_TABLE_NAME_LEN]; } SDropTSMAStmt; #ifdef __cplusplus diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 71d871a775..a8e3343ad0 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -146,6 +146,7 @@ const char *cfgDtypeStr(ECfgDataType type); int32_t cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen); int32_t cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen); +int32_t cfgDumpItemCategory(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen); void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump); void cfgDumpCfgS3(SConfig *pCfg, bool tsc, bool dump); diff --git a/include/util/tdef.h b/include/util/tdef.h index c69d4f8f19..246d1a293d 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -190,45 +190,22 @@ typedef enum EOperatorType { } EOperatorType; static const EOperatorType OPERATOR_ARRAY[] = { - OP_TYPE_ADD, - OP_TYPE_SUB, - OP_TYPE_MULTI, - OP_TYPE_DIV, - OP_TYPE_REM, + OP_TYPE_ADD, OP_TYPE_SUB, OP_TYPE_MULTI, OP_TYPE_DIV, OP_TYPE_REM, - OP_TYPE_MINUS, + OP_TYPE_MINUS, - OP_TYPE_BIT_AND, - OP_TYPE_BIT_OR, + OP_TYPE_BIT_AND, OP_TYPE_BIT_OR, - OP_TYPE_GREATER_THAN, - OP_TYPE_GREATER_EQUAL, - OP_TYPE_LOWER_THAN, - OP_TYPE_LOWER_EQUAL, - OP_TYPE_EQUAL, - OP_TYPE_NOT_EQUAL, - OP_TYPE_IN, - OP_TYPE_NOT_IN, - OP_TYPE_LIKE, - OP_TYPE_NOT_LIKE, - OP_TYPE_MATCH, - OP_TYPE_NMATCH, + OP_TYPE_GREATER_THAN, OP_TYPE_GREATER_EQUAL, OP_TYPE_LOWER_THAN, OP_TYPE_LOWER_EQUAL, OP_TYPE_EQUAL, + OP_TYPE_NOT_EQUAL, OP_TYPE_IN, OP_TYPE_NOT_IN, OP_TYPE_LIKE, OP_TYPE_NOT_LIKE, OP_TYPE_MATCH, OP_TYPE_NMATCH, - OP_TYPE_IS_NULL, - OP_TYPE_IS_NOT_NULL, - OP_TYPE_IS_TRUE, - OP_TYPE_IS_FALSE, - OP_TYPE_IS_UNKNOWN, - OP_TYPE_IS_NOT_TRUE, - OP_TYPE_IS_NOT_FALSE, - OP_TYPE_IS_NOT_UNKNOWN, - //OP_TYPE_COMPARE_MAX_VALUE, + OP_TYPE_IS_NULL, OP_TYPE_IS_NOT_NULL, OP_TYPE_IS_TRUE, OP_TYPE_IS_FALSE, OP_TYPE_IS_UNKNOWN, OP_TYPE_IS_NOT_TRUE, + OP_TYPE_IS_NOT_FALSE, OP_TYPE_IS_NOT_UNKNOWN, + // OP_TYPE_COMPARE_MAX_VALUE, - OP_TYPE_JSON_GET_VALUE, - OP_TYPE_JSON_CONTAINS, + OP_TYPE_JSON_GET_VALUE, OP_TYPE_JSON_CONTAINS, - OP_TYPE_ASSIGN -}; + OP_TYPE_ASSIGN}; #define OP_TYPE_CALC_MAX OP_TYPE_BIT_OR @@ -638,12 +615,13 @@ enum { RAND_ERR_MEMORY = 1, RAND_ERR_FILE = 2, RAND_ERR_NETWORK = 4 }; #define VNODE_HANDLE -3 #define CLIENT_HANDLE -5 -#define TSDB_CONFIG_OPTION_LEN 32 -#define TSDB_CONFIG_VALUE_LEN 64 -#define TSDB_CONFIG_SCOPE_LEN 8 -#define TSDB_CONFIG_NUMBER 16 -#define TSDB_CONFIG_PATH_LEN 4096 -#define TSDB_CONFIG_INFO_LEN 64 +#define TSDB_CONFIG_OPTION_LEN 32 +#define TSDB_CONFIG_VALUE_LEN 64 +#define TSDB_CONFIG_SCOPE_LEN 8 +#define TSDB_CONFIG_NUMBER 16 +#define TSDB_CONFIG_PATH_LEN 4096 +#define TSDB_CONFIG_INFO_LEN 64 +#define TSDB_CONFIG_CATEGORY_LEN 8 #define QUERY_ID_SIZE 20 #define QUERY_OBJ_ID_SIZE 18 diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index da56dcf75e..985c1a4c5a 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -47,11 +47,12 @@ enum { RES_TYPE__TMQ_BATCH_META, }; -#define SHOW_VARIABLES_RESULT_COLS 4 +#define SHOW_VARIABLES_RESULT_COLS 5 #define SHOW_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE) #define SHOW_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE) #define SHOW_VARIABLES_RESULT_FIELD3_LEN (TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE) -#define SHOW_VARIABLES_RESULT_FIELD4_LEN (TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE) +#define SHOW_VARIABLES_RESULT_FIELD4_LEN (TSDB_CONFIG_CATEGORY_LEN + VARSTR_HEADER_SIZE) +#define SHOW_VARIABLES_RESULT_FIELD5_LEN (TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE) #define TD_RES_QUERY(res) (*(int8_t*)(res) == RES_TYPE__QUERY) #define TD_RES_TMQ(res) (*(int8_t*)(res) == RES_TYPE__TMQ) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index e182cd97ee..58ba39864d 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -15,9 +15,10 @@ #include "catalog.h" #include "clientInt.h" -#include "clientMonitor.h" #include "clientLog.h" +#include "clientMonitor.h" #include "cmdnodes.h" +#include "command.h" #include "os.h" #include "query.h" #include "systable.h" @@ -26,7 +27,6 @@ #include "tglobal.h" #include "tname.h" #include "tversion.h" -#include "command.h" extern SClientHbMgr clientHbMgr; @@ -40,7 +40,7 @@ int32_t genericRspCallback(void* param, SDataBuf* pMsg, int32_t code) { setErrno(pRequest, code); if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type)) { - if (removeMeta(pRequest->pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type)) != 0){ + if (removeMeta(pRequest->pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type)) != 0) { tscError("failed to remove meta data for table"); } } @@ -50,7 +50,7 @@ int32_t genericRspCallback(void* param, SDataBuf* pMsg, int32_t code) { if (pRequest->body.queryFp != NULL) { doRequestCallback(pRequest, code); } else { - if (tsem_post(&pRequest->body.rspSem) != 0){ + if (tsem_post(&pRequest->body.rspSem) != 0) { tscError("failed to post semaphore"); } } @@ -104,7 +104,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { SEpSet dstEpSet = connectRsp.epSet; if (srcEpSet.numOfEps == 1) { if (rpcSetDefaultAddr(pTscObj->pAppInfo->pTransporter, srcEpSet.eps[srcEpSet.inUse].fqdn, - dstEpSet.eps[dstEpSet.inUse].fqdn) != 0){ + dstEpSet.eps[dstEpSet.inUse].fqdn) != 0) { tscError("failed to set default addr for rpc"); } updateEpSet = 0; @@ -146,14 +146,15 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { pTscObj->authVer = connectRsp.authVer; pTscObj->whiteListInfo.ver = connectRsp.whiteListVer; - if(taosHashGet(appInfo.pInstMapByClusterId, &connectRsp.clusterId, LONG_BYTES) == NULL){ - if(taosHashPut(appInfo.pInstMapByClusterId, &connectRsp.clusterId, LONG_BYTES, &pTscObj->pAppInfo, POINTER_BYTES) != 0){ + if (taosHashGet(appInfo.pInstMapByClusterId, &connectRsp.clusterId, LONG_BYTES) == NULL) { + if (taosHashPut(appInfo.pInstMapByClusterId, &connectRsp.clusterId, LONG_BYTES, &pTscObj->pAppInfo, + POINTER_BYTES) != 0) { tscError("failed to put appInfo into appInfo.pInstMapByClusterId"); - }else{ + } else { MonitorSlowLogData data = {0}; data.clusterId = pTscObj->pAppInfo->clusterId; data.type = SLOW_LOG_READ_BEGINNIG; - (void)monitorPutData2MonitorQueue(data); // ignore + (void)monitorPutData2MonitorQueue(data); // ignore monitorClientSlowQueryInit(connectRsp.clusterId); monitorClientSQLReqInit(connectRsp.clusterId); } @@ -162,7 +163,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { (void)taosThreadMutexLock(&clientHbMgr.lock); SAppHbMgr* pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, pTscObj->appHbMgrIdx); if (pAppHbMgr) { - if (hbRegisterConn(pAppHbMgr, pTscObj->id, connectRsp.clusterId, connectRsp.connType) != 0){ + if (hbRegisterConn(pAppHbMgr, pTscObj->id, connectRsp.clusterId, connectRsp.connType) != 0) { tscError("0x%" PRIx64 " failed to register conn to hbMgr", pRequest->requestId); } } else { @@ -176,10 +177,10 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { pTscObj->pAppInfo->numOfConns); End: - if (code != 0){ + if (code != 0) { setErrno(pRequest, code); } - if (tsem_post(&pRequest->body.rspSem) != 0){ + if (tsem_post(&pRequest->body.rspSem) != 0) { tscError("failed to post semaphore"); } @@ -196,7 +197,7 @@ EXIT: SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pRequest) { SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if(pMsgSendInfo == NULL) return pMsgSendInfo; + if (pMsgSendInfo == NULL) return pMsgSendInfo; pMsgSendInfo->requestObjRefId = pRequest->self; pMsgSendInfo->requestId = pRequest->requestId; pMsgSendInfo->param = pRequest; @@ -227,11 +228,11 @@ int32_t processCreateDbRsp(void* param, SDataBuf* pMsg, int32_t code) { .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; char dbFName[TSDB_DB_FNAME_LEN]; (void)snprintf(dbFName, sizeof(dbFName) - 1, "%d.%s", pTscObj->acctId, TSDB_INFORMATION_SCHEMA_DB); - if (catalogRefreshDBVgInfo(pCatalog, &conn, dbFName) != 0){ + if (catalogRefreshDBVgInfo(pCatalog, &conn, dbFName) != 0) { tscError("0x%" PRIx64 " failed to refresh db vg info", pRequest->requestId); } (void)snprintf(dbFName, sizeof(dbFName) - 1, "%d.%s", pTscObj->acctId, TSDB_PERFORMANCE_SCHEMA_DB); - if (catalogRefreshDBVgInfo(pCatalog, &conn, dbFName) != 0){ + if (catalogRefreshDBVgInfo(pCatalog, &conn, dbFName) != 0) { tscError("0x%" PRIx64 " failed to refresh db vg info", pRequest->requestId); } } @@ -240,7 +241,7 @@ int32_t processCreateDbRsp(void* param, SDataBuf* pMsg, int32_t code) { if (pRequest->body.queryFp) { doRequestCallback(pRequest, code); } else { - if (tsem_post(&pRequest->body.rspSem) != 0){ + if (tsem_post(&pRequest->body.rspSem) != 0) { tscError("failed to post semaphore"); } } @@ -252,19 +253,19 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { if (TSDB_CODE_MND_DB_NOT_EXIST == code || TSDB_CODE_MND_DB_IN_CREATING == code || TSDB_CODE_MND_DB_IN_DROPPING == code) { SUseDbRsp usedbRsp = {0}; - if (tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp) != 0){ + if (tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp) != 0) { tscError("0x%" PRIx64 " deserialize SUseDbRsp failed", pRequest->requestId); } struct SCatalog* pCatalog = NULL; if (usedbRsp.vgVersion >= 0) { // cached in local int64_t clusterId = pRequest->pTscObj->pAppInfo->clusterId; - int32_t code1 = catalogGetHandle(clusterId, &pCatalog); + int32_t code1 = catalogGetHandle(clusterId, &pCatalog); if (code1 != TSDB_CODE_SUCCESS) { tscWarn("0x%" PRIx64 "catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pRequest->requestId, clusterId, tstrerror(code1)); } else { - if (catalogRemoveDB(pCatalog, usedbRsp.db, usedbRsp.uid) != 0){ + if (catalogRemoveDB(pCatalog, usedbRsp.db, usedbRsp.uid) != 0) { tscError("0x%" PRIx64 "catalogRemoveDB failed, db:%s, uid:%" PRId64, pRequest->requestId, usedbRsp.db, usedbRsp.uid); } @@ -282,7 +283,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { doRequestCallback(pRequest, pRequest->code); } else { - if (tsem_post(&pRequest->body.rspSem) != 0){ + if (tsem_post(&pRequest->body.rspSem) != 0) { tscError("failed to post semaphore"); } } @@ -291,7 +292,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { } SUseDbRsp usedbRsp = {0}; - if (tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp) != 0){ + if (tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp) != 0) { tscError("0x%" PRIx64 " deserialize SUseDbRsp failed", pRequest->requestId); } @@ -309,7 +310,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { tscTrace("db:%s, usedbRsp received, numOfVgroups:%d", usedbRsp.db, usedbRsp.vgNum); for (int32_t i = 0; i < usedbRsp.vgNum; ++i) { SVgroupInfo* pInfo = taosArrayGet(usedbRsp.pVgroupInfos, i); - if (pInfo == NULL){ + if (pInfo == NULL) { continue; } tscTrace("vgId:%d, numOfEps:%d inUse:%d ", pInfo->vgId, pInfo->epSet.numOfEps, pInfo->epSet.inUse); @@ -319,7 +320,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { } SName name = {0}; - if(tNameFromString(&name, usedbRsp.db, T_NAME_ACCT | T_NAME_DB) != TSDB_CODE_SUCCESS) { + if (tNameFromString(&name, usedbRsp.db, T_NAME_ACCT | T_NAME_DB) != TSDB_CODE_SUCCESS) { tscError("0x%" PRIx64 " failed to parse db name:%s", pRequest->requestId, usedbRsp.db); } @@ -338,7 +339,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pRequest->pTscObj->pAppInfo->clusterId, tstrerror(code1)); } else { - if (catalogUpdateDBVgInfo(pCatalog, output.db, output.dbId, output.dbVgroup) != 0){ + if (catalogUpdateDBVgInfo(pCatalog, output.db, output.dbId, output.dbVgroup) != 0) { tscError("0x%" PRIx64 " failed to update db vg info, db:%s, dbId:%" PRId64, pRequest->requestId, output.db, output.dbId); } @@ -350,7 +351,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { tFreeSUsedbRsp(&usedbRsp); char db[TSDB_DB_NAME_LEN] = {0}; - if(tNameGetDbName(&name, db) != TSDB_CODE_SUCCESS) { + if (tNameGetDbName(&name, db) != TSDB_CODE_SUCCESS) { tscError("0x%" PRIx64 " failed to get db name since %s", pRequest->requestId, tstrerror(code)); } @@ -362,7 +363,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { if (pRequest->body.queryFp != NULL) { doRequestCallback(pRequest, pRequest->code); } else { - if (tsem_post(&pRequest->body.rspSem) != 0){ + if (tsem_post(&pRequest->body.rspSem) != 0) { tscError("failed to post semaphore"); } } @@ -387,7 +388,7 @@ int32_t processCreateSTableRsp(void* param, SDataBuf* pMsg, int32_t code) { SMCreateStbRsp createRsp = {0}; SDecoder coder = {0}; tDecoderInit(&coder, pMsg->pData, pMsg->len); - if (pMsg->len > 0){ + if (pMsg->len > 0) { code = tDecodeSMCreateStbRsp(&coder, &createRsp); // pMsg->len == 0 if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); @@ -419,7 +420,7 @@ int32_t processCreateSTableRsp(void* param, SDataBuf* pMsg, int32_t code) { doRequestCallback(pRequest, code); } else { - if (tsem_post(&pRequest->body.rspSem) != 0){ + if (tsem_post(&pRequest->body.rspSem) != 0) { tscError("failed to post semaphore"); } } @@ -432,13 +433,13 @@ int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) { setErrno(pRequest, code); } else { SDropDbRsp dropdbRsp = {0}; - if (tDeserializeSDropDbRsp(pMsg->pData, pMsg->len, &dropdbRsp) != 0){ + if (tDeserializeSDropDbRsp(pMsg->pData, pMsg->len, &dropdbRsp) != 0) { tscError("0x%" PRIx64 " deserialize SDropDbRsp failed", pRequest->requestId); } struct SCatalog* pCatalog = NULL; code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); if (TSDB_CODE_SUCCESS == code) { - if (catalogRemoveDB(pCatalog, dropdbRsp.db, dropdbRsp.uid) != 0){ + if (catalogRemoveDB(pCatalog, dropdbRsp.db, dropdbRsp.uid) != 0) { tscError("0x%" PRIx64 " failed to remove db:%s", pRequest->requestId, dropdbRsp.db); } STscObj* pTscObj = pRequest->pTscObj; @@ -451,7 +452,7 @@ int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) { (void)snprintf(dbFName, sizeof(dbFName) - 1, "%d.%s", pTscObj->acctId, TSDB_INFORMATION_SCHEMA_DB); if (catalogRefreshDBVgInfo(pCatalog, &conn, dbFName) != TSDB_CODE_SUCCESS) { tscError("0x%" PRIx64 " failed to refresh db vg info, db:%s", pRequest->requestId, dbFName); - } + } (void)snprintf(dbFName, sizeof(dbFName) - 1, "%d.%s", pTscObj->acctId, TSDB_PERFORMANCE_SCHEMA_DB); if (catalogRefreshDBVgInfo(pCatalog, &conn, dbFName) != 0) { tscError("0x%" PRIx64 " failed to refresh db vg info, db:%s", pRequest->requestId, dbFName); @@ -465,7 +466,7 @@ int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) { if (pRequest->body.queryFp != NULL) { doRequestCallback(pRequest, code); } else { - if (tsem_post(&pRequest->body.rspSem) != 0){ + if (tsem_post(&pRequest->body.rspSem) != 0) { tscError("failed to post semaphore"); } } @@ -480,7 +481,7 @@ int32_t processAlterStbRsp(void* param, SDataBuf* pMsg, int32_t code) { SMAlterStbRsp alterRsp = {0}; SDecoder coder = {0}; tDecoderInit(&coder, pMsg->pData, pMsg->len); - if (pMsg->len > 0){ + if (pMsg->len > 0) { code = tDecodeSMAlterStbRsp(&coder, &alterRsp); // pMsg->len == 0 if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); @@ -512,7 +513,7 @@ int32_t processAlterStbRsp(void* param, SDataBuf* pMsg, int32_t code) { doRequestCallback(pRequest, code); } else { - if (tsem_post(&pRequest->body.rspSem) != 0){ + if (tsem_post(&pRequest->body.rspSem) != 0) { tscError("failed to post semaphore"); } } @@ -520,8 +521,8 @@ int32_t processAlterStbRsp(void* param, SDataBuf* pMsg, int32_t code) { } static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) { - int32_t code = 0; - int32_t line = 0; + int32_t code = 0; + int32_t line = 0; SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); TSDB_CHECK_NULL(pBlock, code, line, END, terrno); pBlock->info.hasVarCol = true; @@ -545,6 +546,10 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) { infoData.info.bytes = SHOW_VARIABLES_RESULT_FIELD4_LEN; TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, terrno); + infoData.info.type = TSDB_DATA_TYPE_VARCHAR; + infoData.info.bytes = SHOW_VARIABLES_RESULT_FIELD5_LEN; + TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, terrno); + int32_t numOfCfg = taosArrayGetSize(pVars); code = blockDataEnsureCapacity(pBlock, numOfCfg); TSDB_CHECK_CODE(code, line, END); @@ -574,6 +579,13 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) { code = colDataSetVal(pColInfo, i, scope, false); TSDB_CHECK_CODE(code, line, END); + char category[TSDB_CONFIG_CATEGORY_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(category, pInfo->category, TSDB_CONFIG_CATEGORY_LEN + VARSTR_HEADER_SIZE); + pColInfo = taosArrayGet(pBlock->pDataBlock, c++); + TSDB_CHECK_NULL(pColInfo, code, line, END, terrno); + code = colDataSetVal(pColInfo, i, category, false); + TSDB_CHECK_CODE(code, line, END); + char info[TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(info, pInfo->info, TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE); pColInfo = taosArrayGet(pBlock->pDataBlock, c++); @@ -605,7 +617,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { *pRsp = taosMemoryCalloc(1, rspSize); if (NULL == *pRsp) { code = terrno; - goto _exit; + goto _exit; } (*pRsp)->useconds = 0; @@ -617,7 +629,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, dataEncodeBufSize, SHOW_VARIABLES_RESULT_COLS); - if(len < 0) { + if (len < 0) { uError("buildShowVariablesRsp error, len:%d", len); code = terrno; goto _exit; @@ -639,11 +651,11 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { return TSDB_CODE_SUCCESS; _exit: - if(*pRsp) { + if (*pRsp) { taosMemoryFree(*pRsp); *pRsp = NULL; } - if(pBlock) { + if (pBlock) { blockDataDestroy(pBlock); pBlock = NULL; } @@ -677,7 +689,7 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) { if (pRequest->body.queryFp != NULL) { doRequestCallback(pRequest, code); } else { - if (tsem_post(&pRequest->body.rspSem) != 0){ + if (tsem_post(&pRequest->body.rspSem) != 0) { tscError("failed to post semaphore"); } } @@ -685,8 +697,8 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) { } static int32_t buildCompactDbBlock(SCompactDbRsp* pRsp, SSDataBlock** block) { - int32_t code = 0; - int32_t line = 0; + int32_t code = 0; + int32_t line = 0; SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); TSDB_CHECK_NULL(pBlock, code, line, END, terrno); pBlock->info.hasVarCol = true; @@ -772,7 +784,7 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr (*pRsp)->numOfCols = htonl(COMPACT_DB_RESULT_COLS); int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, dataEncodeBufSize, COMPACT_DB_RESULT_COLS); - if(len < 0) { + if (len < 0) { uError("buildRetriveTableRspForCompactDb error, len:%d", len); code = terrno; goto _exit; @@ -794,24 +806,23 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr return TSDB_CODE_SUCCESS; _exit: - if(*pRsp) { + if (*pRsp) { taosMemoryFree(*pRsp); *pRsp = NULL; } - if(pBlock) { + if (pBlock) { blockDataDestroy(pBlock); pBlock = NULL; } return code; } - int32_t processCompactDbRsp(void* param, SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); } else { - SCompactDbRsp rsp = {0}; + SCompactDbRsp rsp = {0}; SRetrieveTableRsp* pRes = NULL; code = tDeserializeSCompactDbRsp(pMsg->pData, pMsg->len, &rsp); if (TSDB_CODE_SUCCESS == code) { @@ -830,9 +841,9 @@ int32_t processCompactDbRsp(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFree(pMsg->pEpSet); if (pRequest->body.queryFp != NULL) { - pRequest->body.queryFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, code); + pRequest->body.queryFp(((SSyncQueryParam*)pRequest->body.interParam)->userParam, pRequest, code); } else { - if (tsem_post(&pRequest->body.rspSem) != 0){ + if (tsem_post(&pRequest->body.rspSem) != 0) { tscError("failed to post semaphore"); } } diff --git a/source/common/src/msg/tmsg.c b/source/common/src/msg/tmsg.c index ca94fd8f38..2fa9c50233 100644 --- a/source/common/src/msg/tmsg.c +++ b/source/common/src/msg/tmsg.c @@ -77,7 +77,7 @@ static int32_t tSerializeSMonitorParas(SEncoder *encoder, const SMonitorParas *p TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogScope)); TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogMaxLen)); TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogThreshold)); - TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogThresholdTest)); //Obsolete + TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogThresholdTest)); // Obsolete TAOS_CHECK_RETURN(tEncodeCStr(encoder, pMonitorParas->tsSlowLogExceptDb)); return 0; } @@ -88,7 +88,7 @@ static int32_t tDeserializeSMonitorParas(SDecoder *decoder, SMonitorParas *pMoni TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogScope)); TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogMaxLen)); TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogThreshold)); - TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogThresholdTest)); //Obsolete + TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogThresholdTest)); // Obsolete TAOS_CHECK_RETURN(tDecodeCStrTo(decoder, pMonitorParas->tsSlowLogExceptDb)); return 0; } @@ -2255,9 +2255,9 @@ int32_t tSerializeRetrieveAnalAlgoRsp(void *buf, int32_t bufLen, SRetrieveAnalAl int32_t numOfAlgos = 0; void *pIter = taosHashIterate(pRsp->hash, NULL); while (pIter != NULL) { - SAnalyticsUrl *pUrl = pIter; - size_t nameLen = 0; - const char *name = taosHashGetKey(pIter, &nameLen); + SAnalyticsUrl *pUrl = pIter; + size_t nameLen = 0; + const char *name = taosHashGetKey(pIter, &nameLen); if (nameLen > 0 && nameLen <= TSDB_ANALYTIC_ALGO_KEY_LEN && pUrl->urlLen > 0) { numOfAlgos++; } @@ -2270,9 +2270,9 @@ int32_t tSerializeRetrieveAnalAlgoRsp(void *buf, int32_t bufLen, SRetrieveAnalAl pIter = taosHashIterate(pRsp->hash, NULL); while (pIter != NULL) { - SAnalyticsUrl *pUrl = pIter; - size_t nameLen = 0; - const char *name = taosHashGetKey(pIter, &nameLen); + SAnalyticsUrl *pUrl = pIter; + size_t nameLen = 0; + const char *name = taosHashGetKey(pIter, &nameLen); if (nameLen > 0 && pUrl->urlLen > 0) { TAOS_CHECK_EXIT(tEncodeI32(&encoder, nameLen)); TAOS_CHECK_EXIT(tEncodeBinary(&encoder, (const uint8_t *)name, nameLen)); @@ -2310,11 +2310,10 @@ int32_t tDeserializeRetrieveAnalAlgoRsp(void *buf, int32_t bufLen, SRetrieveAnal int32_t lino; tDecoderInit(&decoder, buf, bufLen); - - int32_t numOfAlgos = 0; - int32_t nameLen; - int32_t type; - char name[TSDB_ANALYTIC_ALGO_KEY_LEN]; + int32_t numOfAlgos = 0; + int32_t nameLen; + int32_t type; + char name[TSDB_ANALYTIC_ALGO_KEY_LEN]; SAnalyticsUrl url = {0}; TAOS_CHECK_EXIT(tStartDecode(&decoder)); @@ -5710,6 +5709,7 @@ int32_t tEncodeSVariablesInfo(SEncoder *pEncoder, SVariablesInfo *pInfo) { TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pInfo->name)); TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pInfo->value)); TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pInfo->scope)); + TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pInfo->category)); return 0; } @@ -5717,6 +5717,7 @@ int32_t tDecodeSVariablesInfo(SDecoder *pDecoder, SVariablesInfo *pInfo) { TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pInfo->name)); TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pInfo->value)); TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pInfo->scope)); + TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pInfo->category)); return 0; } diff --git a/source/common/src/systable.c b/source/common/src/systable.c index bfe82aa7ae..e3d119f198 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -329,6 +329,7 @@ static const SSysDbTableSchema variablesSchema[] = { {.name = "name", .bytes = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "value", .bytes = TSDB_CONFIG_PATH_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "scope", .bytes = TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "category", .bytes = TSDB_CONFIG_CATEGORY_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "info", .bytes = TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 01966c0536..39c5b03a10 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -2709,7 +2709,7 @@ int32_t globalConfigSerialize(int32_t version, SArray *array, char **serialized) char buf[30]; cJSON *json = cJSON_CreateObject(); if (json == NULL) goto _exit; - json = cJSON_AddNumberToObject(json, "version", version); + if (cJSON_AddNumberToObject(json, "version", version) == NULL) goto _exit; if (json == NULL) goto _exit; int sz = taosArrayGetSize(array); @@ -2725,38 +2725,31 @@ int32_t globalConfigSerialize(int32_t version, SArray *array, char **serialized) case CFG_DTYPE_NONE: break; case CFG_DTYPE_BOOL: - cField = cJSON_AddBoolToObject(cField, item->name, item->bval); - if (cField == NULL) goto _exit; + if (cJSON_AddBoolToObject(cField, item->name, item->bval) == NULL) goto _exit; break; case CFG_DTYPE_INT32: - cField = cJSON_AddNumberToObject(cField, item->name, item->i32); - if (cField == NULL) goto _exit; + if (cJSON_AddNumberToObject(cField, item->name, item->i32) == NULL) goto _exit; break; case CFG_DTYPE_INT64: (void)sprintf(buf, "%" PRId64, item->i64); - cField = cJSON_AddStringToObject(cField, item->name, buf); - if (cField == NULL) goto _exit; + if (cJSON_AddStringToObject(cField, item->name, buf) == NULL) goto _exit; break; case CFG_DTYPE_FLOAT: case CFG_DTYPE_DOUBLE: (void)sprintf(buf, "%f", item->fval); - cField = cJSON_AddStringToObject(cField, item->name, buf); - if (cField == NULL) goto _exit; + if (cJSON_AddStringToObject(cField, item->name, buf) == NULL) goto _exit; break; case CFG_DTYPE_STRING: case CFG_DTYPE_DIR: case CFG_DTYPE_LOCALE: case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: - cField = cJSON_AddStringToObject(cField, item->name, item->str); - if (cField == NULL) goto _exit; + if (cJSON_AddStringToObject(cField, item->name, item->str) == NULL) goto _exit; break; } } } - if (!cJSON_AddItemToObject(json, "configs", cField)) { - goto _exit; - } + if (!cJSON_AddItemToObject(json, "configs", cField)) goto _exit; *serialized = cJSON_Print(json); _exit: if (terrno != TSDB_CODE_SUCCESS) { @@ -2784,31 +2777,26 @@ int32_t localConfigSerialize(SArray *array, char **serialized) { case CFG_DTYPE_NONE: break; case CFG_DTYPE_BOOL: - cField = cJSON_AddBoolToObject(cField, item->name, item->bval); - if (cField == NULL) goto _exit; + if (cJSON_AddBoolToObject(cField, item->name, item->bval) == NULL) goto _exit; break; case CFG_DTYPE_INT32: - cField = cJSON_AddNumberToObject(cField, item->name, item->i32); - if (cField == NULL) goto _exit; + if (cJSON_AddNumberToObject(cField, item->name, item->i32) == NULL) goto _exit; break; case CFG_DTYPE_INT64: (void)sprintf(buf, "%" PRId64, item->i64); - cField = cJSON_AddStringToObject(cField, item->name, buf); - if (cField == NULL) goto _exit; + if (cJSON_AddStringToObject(cField, item->name, buf) == NULL) goto _exit; break; case CFG_DTYPE_FLOAT: case CFG_DTYPE_DOUBLE: (void)sprintf(buf, "%f", item->fval); - cField = cJSON_AddStringToObject(cField, item->name, buf); - if (cField == NULL) goto _exit; + if (cJSON_AddStringToObject(cField, item->name, buf) == NULL) goto _exit; break; case CFG_DTYPE_STRING: case CFG_DTYPE_DIR: case CFG_DTYPE_LOCALE: case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: - cField = cJSON_AddStringToObject(cField, item->name, item->str); - if (cField == NULL) goto _exit; + if (cJSON_AddStringToObject(cField, item->name, item->str) == NULL) goto _exit; break; } } @@ -2846,7 +2834,7 @@ int32_t taosPersistGlobalConfig(SArray *array, const char *path, int32_t version char *serialized = NULL; TAOS_CHECK_GOTO(globalConfigSerialize(version, array, &serialized), &lino, _exit); - if (taosWriteFile(pConfigFile, serialized, strlen(serialized) < 0)) { + if (taosWriteFile(pConfigFile, serialized, strlen(serialized)) < 0) { lino = __LINE__; code = TAOS_SYSTEM_ERROR(errno); uError("failed to write file:%s since %s", filename, tstrerror(code)); diff --git a/source/common/src/tmisce.c b/source/common/src/tmisce.c index 8988fab56a..8598645ea5 100644 --- a/source/common/src/tmisce.c +++ b/source/common/src/tmisce.c @@ -14,10 +14,10 @@ */ #define _DEFAULT_SOURCE -#include "tmisce.h" #include "tdatablock.h" #include "tglobal.h" #include "tjson.h" +#include "tmisce.h" int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) { pEp->port = 0; @@ -282,7 +282,7 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { locked = 1; while ((pItem = cfgNextIter(pIter)) != NULL) { -_start: + _start: col = startCol; // GRANT_CFG_SKIP; @@ -297,11 +297,11 @@ _start: TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, name, false), NULL, _exit); - char value[TSDB_CONFIG_PATH_LEN + VARSTR_HEADER_SIZE] = {0}; - int32_t valueLen = 0; + char value[TSDB_CONFIG_PATH_LEN + VARSTR_HEADER_SIZE] = {0}; + int32_t valueLen = 0; SDiskCfg* pDiskCfg = NULL; if (strcasecmp(pItem->name, "dataDir") == 0 && exSize > 0) { - char* buf = &value[VARSTR_HEADER_SIZE]; + char* buf = &value[VARSTR_HEADER_SIZE]; pDiskCfg = taosArrayGet(pItem->array, index); valueLen = tsnprintf(buf, TSDB_CONFIG_PATH_LEN, "%s", pDiskCfg->dir); index++; @@ -330,11 +330,23 @@ _start: } TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, scope, false), NULL, _exit); + char category[TSDB_CONFIG_CATEGORY_LEN + VARSTR_HEADER_SIZE] = {0}; + TAOS_CHECK_GOTO(cfgDumpItemCategory(pItem, &category[VARSTR_HEADER_SIZE], TSDB_CONFIG_CATEGORY_LEN, &valueLen), + NULL, _exit); + varDataSetLen(category, valueLen); + + pColInfo = taosArrayGet(pBlock->pDataBlock, col++); + if (pColInfo == NULL) { + code = terrno; + TAOS_CHECK_GOTO(code, NULL, _exit); + } + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, category, false), NULL, _exit); + char info[TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE] = {0}; if (strcasecmp(pItem->name, "dataDir") == 0 && pDiskCfg) { char* buf = &info[VARSTR_HEADER_SIZE]; valueLen = tsnprintf(buf, TSDB_CONFIG_INFO_LEN, "level %d primary %d disabled %" PRIi8, pDiskCfg->level, - pDiskCfg->primary, pDiskCfg->disable); + pDiskCfg->primary, pDiskCfg->disable); } else { valueLen = 0; } @@ -351,7 +363,7 @@ _start: if (index > 0 && index <= exSize) { goto _start; } -} + } pBlock->info.rows = numOfRows; _exit: if (locked) cfgUnLock(pConf); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 862af15ff3..da9b50e934 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -1079,6 +1079,78 @@ static void getSlowLogScopeString(int32_t scope, char *result) { } } +SArray *initVariablesFromItems(SArray *pItems) { + if (pItems == NULL) { + return NULL; + } + + int32_t sz = taosArrayGetSize(pItems); + + SArray *pInfos = taosArrayInit(sz, sizeof(SVariablesInfo)); + for (int32_t i = 0; i < sz; ++i) { + SConfigItem *pItem = taosArrayGet(pItems, i); + SVariablesInfo info = {0}; + strcpy(info.name, pItem->name); + + // init info value + switch (pItem->dtype) { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + sprintf(info.value, "%d", pItem->bval); + break; + case CFG_DTYPE_INT32: + sprintf(info.value, "%d", pItem->i32); + break; + case CFG_DTYPE_INT64: + sprintf(info.value, "%" PRId64, pItem->i64); + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + sprintf(info.value, "%f", pItem->fval); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + sprintf(info.value, "%s", pItem->str); + break; + } + + // init info scope + switch (pItem->scope) { + case CFG_SCOPE_SERVER: + strcpy(info.scope, "server"); + break; + case CFG_SCOPE_CLIENT: + strcpy(info.scope, "client"); + break; + case CFG_SCOPE_BOTH: + strcpy(info.scope, "both"); + break; + default: + strcpy(info.scope, "unknown"); + break; + } + // init info category + switch (pItem->category) { + case CFG_CATEGORY_GLOBAL: + strcpy(info.category, "global"); + break; + case CFG_CATEGORY_LOCAL: + strcpy(info.category, "local"); + break; + default: + strcpy(info.category, "unknown"); + break; + } + taosArrayPush(pInfos, &info); + } + + return pInfos; +} + static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { SShowVariablesRsp rsp = {0}; int32_t code = -1; @@ -1087,90 +1159,13 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { goto _OVER; } - rsp.variables = taosArrayInit(16, sizeof(SVariablesInfo)); - if (NULL == rsp.variables) { - mError("failed to alloc SVariablesInfo array while process show variables req"); - code = terrno; - goto _OVER; - } - SVariablesInfo info = {0}; - (void)strcpy(info.name, "statusInterval"); - (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%d", tsStatusInterval); - (void)strcpy(info.scope, "server"); - // fill info.info - if (taosArrayPush(rsp.variables, &info) == NULL) { + rsp.variables = initVariablesFromItems(taosGetGlobalCfg(tsCfg)); + if (rsp.variables == NULL) { code = terrno; goto _OVER; } - - (void)strcpy(info.name, "timezone"); - (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%s", tsTimezoneStr); - (void)strcpy(info.scope, "both"); - if (taosArrayPush(rsp.variables, &info) == NULL) { - code = terrno; - goto _OVER; - } - - (void)strcpy(info.name, "locale"); - (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%s", tsLocale); - (void)strcpy(info.scope, "both"); - if (taosArrayPush(rsp.variables, &info) == NULL) { - code = terrno; - goto _OVER; - } - - (void)strcpy(info.name, "charset"); - (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%s", tsCharset); - (void)strcpy(info.scope, "both"); - if (taosArrayPush(rsp.variables, &info) == NULL) { - code = terrno; - goto _OVER; - } - - (void)strcpy(info.name, "monitor"); - (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%d", tsEnableMonitor); - (void)strcpy(info.scope, "server"); - if (taosArrayPush(rsp.variables, &info) == NULL) { - code = terrno; - goto _OVER; - } - - (void)strcpy(info.name, "monitorInterval"); - (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%d", tsMonitorInterval); - (void)strcpy(info.scope, "server"); - if (taosArrayPush(rsp.variables, &info) == NULL) { - code = terrno; - goto _OVER; - } - - (void)strcpy(info.name, "slowLogThreshold"); - (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%d", tsSlowLogThreshold); - (void)strcpy(info.scope, "server"); - if (taosArrayPush(rsp.variables, &info) == NULL) { - code = terrno; - goto _OVER; - } - - (void)strcpy(info.name, "slowLogMaxLen"); - (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%d", tsSlowLogMaxLen); - (void)strcpy(info.scope, "server"); - if (taosArrayPush(rsp.variables, &info) == NULL) { - code = terrno; - goto _OVER; - } - - char scopeStr[64] = {0}; - getSlowLogScopeString(tsSlowLogScope, scopeStr); - (void)strcpy(info.name, "slowLogScope"); - (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%s", scopeStr); - (void)strcpy(info.scope, "server"); - if (taosArrayPush(rsp.variables, &info) == NULL) { - code = terrno; - goto _OVER; - } - int32_t rspLen = tSerializeSShowVariablesRsp(NULL, 0, &rsp); void *pRsp = rpcMallocCont(rspLen); if (pRsp == NULL) { diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 6ea8dd6757..99993277d5 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -223,7 +223,7 @@ static int32_t execDescribe(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** if (NULL == pDesc || NULL == pDesc->pMeta) { return TSDB_CODE_INVALID_PARA; } - int32_t numOfRows = TABLE_TOTAL_COL_NUM(pDesc->pMeta); + int32_t numOfRows = TABLE_TOTAL_COL_NUM(pDesc->pMeta); SSDataBlock* pBlock = NULL; int32_t code = buildDescResultDataBlock(&pBlock); @@ -960,6 +960,12 @@ static int32_t buildLocalVariablesResultDataBlock(SSDataBlock** pOutput) { goto _exit; } + infoData.info.type = TSDB_DATA_TYPE_VARCHAR; + infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD5_LEN; + if (taosArrayPush(pBlock->pDataBlock, &infoData) == NULL) { + goto _exit; + } + *pOutput = pBlock; _exit: diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index fcb6361a6b..57530b788d 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "parTranslater.h" #include "parInt.h" +#include "parTranslater.h" #include "tdatablock.h" #include "catalog.h" @@ -1853,7 +1853,7 @@ static bool clauseSupportAlias(ESqlClause clause) { return SQL_CLAUSE_GROUP_BY == clause || SQL_CLAUSE_PARTITION_BY == clause || SQL_CLAUSE_ORDER_BY == clause; } -static EDealRes translateColumnInGroupByClause(STranslateContext* pCxt, SColumnNode** pCol, bool *translateAsAlias) { +static EDealRes translateColumnInGroupByClause(STranslateContext* pCxt, SColumnNode** pCol, bool* translateAsAlias) { *translateAsAlias = false; // count(*)/first(*)/last(*) and so on if (0 == strcmp((*pCol)->colName, "*")) { @@ -1862,7 +1862,7 @@ static EDealRes translateColumnInGroupByClause(STranslateContext* pCxt, SColumnN if (pCxt->pParseCxt->biMode) { SNode** ppNode = (SNode**)pCol; - bool ret; + bool ret; pCxt->errCode = biRewriteToTbnameFunc(pCxt, ppNode, &ret); if (TSDB_CODE_SUCCESS != pCxt->errCode) return DEAL_RES_ERROR; if (ret) { @@ -1876,9 +1876,8 @@ static EDealRes translateColumnInGroupByClause(STranslateContext* pCxt, SColumnN } else { bool found = false; res = translateColumnWithoutPrefix(pCxt, pCol); - if (!(*pCol)->node.asParam && - res != DEAL_RES_CONTINUE && - res != DEAL_RES_END && pCxt->errCode != TSDB_CODE_PAR_AMBIGUOUS_COLUMN) { + if (!(*pCol)->node.asParam && res != DEAL_RES_CONTINUE && res != DEAL_RES_END && + pCxt->errCode != TSDB_CODE_PAR_AMBIGUOUS_COLUMN) { res = translateColumnUseAlias(pCxt, pCol, &found); *translateAsAlias = true; } @@ -3321,9 +3320,11 @@ static int32_t selectCommonType(SDataType* commonType, const SDataType* newType) return TSDB_CODE_SUCCESS; } - if ((resultType == TSDB_DATA_TYPE_VARCHAR) && (IS_MATHABLE_TYPE(commonType->type) || IS_MATHABLE_TYPE(newType->type))) { + if ((resultType == TSDB_DATA_TYPE_VARCHAR) && + (IS_MATHABLE_TYPE(commonType->type) || IS_MATHABLE_TYPE(newType->type))) { commonType->bytes = TMAX(TMAX(commonType->bytes, newType->bytes), QUERY_NUMBER_MAX_DISPLAY_LEN); - } else if ((resultType == TSDB_DATA_TYPE_NCHAR) && (IS_MATHABLE_TYPE(commonType->type) || IS_MATHABLE_TYPE(newType->type))) { + } else if ((resultType == TSDB_DATA_TYPE_NCHAR) && + (IS_MATHABLE_TYPE(commonType->type) || IS_MATHABLE_TYPE(newType->type))) { commonType->bytes = TMAX(TMAX(commonType->bytes, newType->bytes), QUERY_NUMBER_MAX_DISPLAY_LEN * TSDB_NCHAR_SIZE); } else { commonType->bytes = TMAX(TMAX(commonType->bytes, newType->bytes), TYPE_BYTES[resultType]); @@ -5480,7 +5481,7 @@ static EDealRes translateGroupPartitionByImpl(SNode** pNode, void* pContext) { int32_t code = TSDB_CODE_SUCCESS; STranslateContext* pTransCxt = pCxt->pTranslateCxt; if (QUERY_NODE_VALUE == nodeType(*pNode)) { - SValueNode* pVal = (SValueNode*) *pNode; + SValueNode* pVal = (SValueNode*)*pNode; if (DEAL_RES_ERROR == translateValue(pTransCxt, pVal)) { return DEAL_RES_CONTINUE; } @@ -5528,8 +5529,7 @@ static int32_t translateGroupByList(STranslateContext* pCxt, SSelectStmt* pSelec if (NULL == pSelect->pGroupByList) { return TSDB_CODE_SUCCESS; } - SReplaceGroupByAliasCxt cxt = { - .pTranslateCxt = pCxt, .pProjectionList = pSelect->pProjectionList}; + SReplaceGroupByAliasCxt cxt = {.pTranslateCxt = pCxt, .pProjectionList = pSelect->pProjectionList}; nodesRewriteExprsPostOrder(pSelect->pGroupByList, translateGroupPartitionByImpl, &cxt); return pCxt->errCode; @@ -5540,8 +5540,7 @@ static int32_t translatePartitionByList(STranslateContext* pCxt, SSelectStmt* pS return TSDB_CODE_SUCCESS; } - SReplaceGroupByAliasCxt cxt = { - .pTranslateCxt = pCxt, .pProjectionList = pSelect->pProjectionList}; + SReplaceGroupByAliasCxt cxt = {.pTranslateCxt = pCxt, .pProjectionList = pSelect->pProjectionList}; nodesRewriteExprsPostOrder(pSelect->pPartitionByList, translateGroupPartitionByImpl, &cxt); return pCxt->errCode; @@ -10521,7 +10520,8 @@ static void getSourceDatabase(SNode* pStmt, int32_t acctId, char* pDbFName) { (void)tNameGetFullDbName(&name, pDbFName); } -static void getStreamQueryFirstProjectAliasName(SHashObj* pUserAliasSet, char* aliasName, int32_t len, char* defaultName[]) { +static void getStreamQueryFirstProjectAliasName(SHashObj* pUserAliasSet, char* aliasName, int32_t len, + char* defaultName[]) { for (int32_t i = 0; defaultName[i] != NULL; i++) { if (NULL == taosHashGet(pUserAliasSet, defaultName[i], strlen(defaultName[i]))) { snprintf(aliasName, len, "%s", defaultName[i]); @@ -10547,8 +10547,8 @@ static int32_t setColumnDefNodePrimaryKey(SColumnDefNode* pNode, bool isPk) { return code; } -static int32_t addIrowTsToCreateStreamQueryImpl(STranslateContext* pCxt, SSelectStmt* pSelect, - SHashObj* pUserAliasSet, SNodeList* pCols, SCMCreateStreamReq* pReq) { +static int32_t addIrowTsToCreateStreamQueryImpl(STranslateContext* pCxt, SSelectStmt* pSelect, SHashObj* pUserAliasSet, + SNodeList* pCols, SCMCreateStreamReq* pReq) { SNode* pProj = nodesListGetNode(pSelect->pProjectionList, 0); if (!pSelect->hasInterpFunc || (QUERY_NODE_FUNCTION == nodeType(pProj) && 0 == strcmp("_irowts", ((SFunctionNode*)pProj)->functionName))) { @@ -10595,7 +10595,7 @@ static int32_t addWstartTsToCreateStreamQueryImpl(STranslateContext* pCxt, SSele return TSDB_CODE_SUCCESS; } SFunctionNode* pFunc = NULL; - int32_t code = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&pFunc); + int32_t code = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&pFunc); if (NULL == pFunc) { return code; } @@ -10627,7 +10627,7 @@ static int32_t addWstartTsToCreateStreamQueryImpl(STranslateContext* pCxt, SSele } static int32_t addTsKeyToCreateStreamQuery(STranslateContext* pCxt, SNode* pStmt, SNodeList* pCols, - SCMCreateStreamReq* pReq) { + SCMCreateStreamReq* pReq) { SSelectStmt* pSelect = (SSelectStmt*)pStmt; SHashObj* pUserAliasSet = NULL; int32_t code = checkProjectAlias(pCxt, pSelect->pProjectionList, &pUserAliasSet); @@ -10990,21 +10990,18 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm if (pStmt->pOptions->triggerType == STREAM_TRIGGER_FORCE_WINDOW_CLOSE) { if (pStmt->pOptions->fillHistory) { - return generateSyntaxErrMsgExt( - &pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "When trigger was force window close, Stream unsupported Fill history"); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "When trigger was force window close, Stream unsupported Fill history"); } if (pStmt->pOptions->ignoreExpired != 1) { - return generateSyntaxErrMsgExt( - &pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "When trigger was force window close, Stream must not set ignore expired 0"); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "When trigger was force window close, Stream must not set ignore expired 0"); } if (pStmt->pOptions->ignoreUpdate != 1) { - return generateSyntaxErrMsgExt( - &pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "When trigger was force window close, Stream must not set ignore update 0"); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "When trigger was force window close, Stream must not set ignore update 0"); } if (pSelect->pWindow != NULL && QUERY_NODE_INTERVAL_WINDOW == nodeType(pSelect->pWindow)) { @@ -13127,7 +13124,7 @@ static int32_t extractShowCreateViewResultSchema(int32_t* numOfCols, SSchema** p } static int32_t extractShowVariablesResultSchema(int32_t* numOfCols, SSchema** pSchema) { - *numOfCols = SHOW_LOCAL_VARIABLES_RESULT_COLS; // SHOW_VARIABLES_RESULT_COLS + *numOfCols = SHOW_LOCAL_VARIABLES_RESULT_COLS; // SHOW_VARIABLES_RESULT_COLS *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); if (NULL == (*pSchema)) { return terrno; @@ -13146,8 +13143,12 @@ static int32_t extractShowVariablesResultSchema(int32_t* numOfCols, SSchema** pS strcpy((*pSchema)[2].name, "scope"); (*pSchema)[3].type = TSDB_DATA_TYPE_BINARY; - (*pSchema)[3].bytes = TSDB_CONFIG_INFO_LEN; - strcpy((*pSchema)[3].name, "info"); + (*pSchema)[3].bytes = TSDB_CONFIG_CATEGORY_LEN; + strcpy((*pSchema)[3].name, "category"); + + (*pSchema)[4].type = TSDB_DATA_TYPE_BINARY; + (*pSchema)[4].bytes = TSDB_CONFIG_INFO_LEN; + strcpy((*pSchema)[4].name, "info"); return TSDB_CODE_SUCCESS; } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 478d9516d3..f63a80b5aa 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -822,6 +822,32 @@ int32_t cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t TAOS_RETURN(TSDB_CODE_SUCCESS); } +int32_t cfgDumpItemCategory(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen) { + int32_t len = 0; + switch (pItem->category) { + case CFG_CATEGORY_LOCAL: + len = tsnprintf(buf, bufSize, "local"); + break; + case CFG_CATEGORY_GLOBAL: + len = tsnprintf(buf, bufSize, "global"); + break; + default: + uError("invalid category:%d", pItem->category); + TAOS_RETURN(TSDB_CODE_INVALID_CFG); + } + + if (len < 0) { + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + } + + if (len > bufSize) { + len = bufSize; + } + + *pLen = len; + TAOS_RETURN(TSDB_CODE_SUCCESS); +} + void cfgDumpCfgArrayS3(SArray *array, bool tsc, bool dump) { char src[CFG_SRC_PRINT_LEN + 1] = {0}; char name[CFG_NAME_PRINT_LEN + 1] = {0}; From 223e0f55c5013d8abdb4415bcfdb20b980b4d9ce Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 26 Nov 2024 17:05:07 +0800 Subject: [PATCH 38/99] Fix ci error. --- source/dnode/mnode/impl/src/mndConfig.c | 1 + source/dnode/mnode/impl/src/mndDef.c | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 5 ++++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index 563c14411d..e861f1c0e8 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -66,6 +66,7 @@ int32_t taosGetConfigObjSize(SConfigObj *obj) { if (obj->dtype == CFG_DTYPE_STRING || obj->dtype == CFG_DTYPE_DIR || obj->dtype == CFG_DTYPE_LOCALE || obj->dtype == CFG_DTYPE_CHARSET || obj->dtype == CFG_DTYPE_TIMEZONE) { if (obj->str != NULL) { + size += sizeof(int32_t); size += strlen(obj->str) + 1; } } diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 0ed136e35b..3741a8bae0 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -758,7 +758,7 @@ SConfigObj *mndInitConfigObj(SConfigItem *pItem) { case CFG_DTYPE_LOCALE: case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: - pObj->str = pItem->str; + pObj->str = taosStrdup(pItem->str); break; } return pObj; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index da9b50e934..dfe0c1e933 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -1145,7 +1145,10 @@ SArray *initVariablesFromItems(SArray *pItems) { strcpy(info.category, "unknown"); break; } - taosArrayPush(pInfos, &info); + if (NULL == taosArrayPush(pInfos, &info)) { + mError("failed to push info to array while init variables from items,since %s", tstrerror(terrno)); + return NULL; + } } return pInfos; From c6e39b8286ec5037041eedc01754b7690af11b70 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 26 Nov 2024 19:23:10 +0800 Subject: [PATCH 39/99] fix heap over flow at obj str. --- source/dnode/mnode/impl/src/mndConfig.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index e861f1c0e8..09754b8344 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -112,8 +112,9 @@ SSdbRaw *mnCfgActionEncode(SConfigObj *obj) { case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: if (obj->str != NULL) { - SDB_SET_INT32(pRaw, dataPos, strlen(obj->str), _OVER) - SDB_SET_BINARY(pRaw, dataPos, obj->str, strlen(obj->str), _OVER) + int32_t len = strlen(obj->str) + 1; + SDB_SET_INT32(pRaw, dataPos, len, _OVER) + SDB_SET_BINARY(pRaw, dataPos, obj->str, len, _OVER) } else { SDB_SET_INT32(pRaw, dataPos, 0, _OVER) } @@ -184,7 +185,7 @@ SSdbRow *mndCfgActionDecode(SSdbRaw *pRaw) { case CFG_DTYPE_TIMEZONE: SDB_GET_INT32(pRaw, dataPos, &len, _OVER) if (len > 0) { - obj->str = taosMemoryMalloc(len + 1); + obj->str = taosMemoryMalloc(len); SDB_GET_BINARY(pRaw, dataPos, obj->str, len, _OVER) } break; From 90af4457ef16ecae79959d7494a29b1f3dbd7073 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 27 Nov 2024 10:50:04 +0800 Subject: [PATCH 40/99] Add func sdb set&get float. --- source/dnode/mnode/impl/src/mndConfig.c | 10 ++------ source/dnode/mnode/sdb/inc/sdb.h | 4 ++++ source/dnode/mnode/sdb/src/sdbRaw.c | 32 +++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index 09754b8344..ba53648823 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -102,9 +102,7 @@ SSdbRaw *mnCfgActionEncode(SConfigObj *obj) { break; case CFG_DTYPE_FLOAT: case CFG_DTYPE_DOUBLE: - (void)sprintf(buf, "%f", obj->fval); - SDB_SET_INT32(pRaw, dataPos, strlen(buf), _OVER) - SDB_SET_BINARY(pRaw, dataPos, buf, strlen(buf), _OVER) + SDB_SET_FLOAT(pRaw, dataPos, obj->fval, _OVER) break; case CFG_DTYPE_STRING: case CFG_DTYPE_DIR: @@ -172,11 +170,7 @@ SSdbRow *mndCfgActionDecode(SSdbRaw *pRaw) { break; case CFG_DTYPE_FLOAT: case CFG_DTYPE_DOUBLE: - SDB_GET_INT32(pRaw, dataPos, &len, _OVER) - char *buf = taosMemoryMalloc(len + 1); - SDB_GET_BINARY(pRaw, dataPos, buf, len, _OVER) - obj->fval = atof(buf); - taosMemoryFree(buf); + SDB_GET_FLOAT(pRaw, dataPos, &obj->fval, _OVER) break; case CFG_DTYPE_STRING: case CFG_DTYPE_DIR: diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index 19a9343cea..eff26bf412 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -56,6 +56,7 @@ extern "C" { } #define SDB_GET_INT64(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawInt64, int64_t) +#define SDB_GET_FLOAT(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawFloat, float) #define SDB_GET_INT32(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawInt32, int32_t) #define SDB_GET_INT16(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawInt16, int16_t) #define SDB_GET_INT8(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawInt8, int8_t) @@ -82,6 +83,7 @@ extern "C" { #define SDB_SET_INT16(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawInt16, int16_t) #define SDB_SET_INT8(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawInt8, int8_t) #define SDB_SET_UINT8(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawUInt8, uint8_t) +#define SDB_SET_FLOAT(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawFloat, float) #define SDB_SET_BOOL(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawBool, bool) #define SDB_SET_BINARY(pRaw, dataPos, val, valLen, pos) \ @@ -425,6 +427,7 @@ int32_t sdbSetRawBool(SSdbRaw *pRaw, int32_t dataPos, bool val); int32_t sdbSetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t val); int32_t sdbSetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t val); int32_t sdbSetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t val); +int32_t sdbSetRawFloat(SSdbRaw *pRaw, int32_t dataPos, float val); int32_t sdbSetRawBinary(SSdbRaw *pRaw, int32_t dataPos, const char *pVal, int32_t valLen); int32_t sdbSetRawDataLen(SSdbRaw *pRaw, int32_t dataLen); int32_t sdbSetRawStatus(SSdbRaw *pRaw, ESdbStatus status); @@ -434,6 +437,7 @@ int32_t sdbGetRawBool(SSdbRaw *pRaw, int32_t dataPos, bool *val); int32_t sdbGetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t *val); int32_t sdbGetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t *val); int32_t sdbGetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t *val); +int32_t sdbGetRawFloat(SSdbRaw *pRaw, int32_t dataPos, float *val); int32_t sdbGetRawBinary(SSdbRaw *pRaw, int32_t dataPos, char *pVal, int32_t valLen); int32_t sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver); int32_t sdbGetRawTotalSize(SSdbRaw *pRaw); diff --git a/source/dnode/mnode/sdb/src/sdbRaw.c b/source/dnode/mnode/sdb/src/sdbRaw.c index c71e415d1b..37131300c6 100644 --- a/source/dnode/mnode/sdb/src/sdbRaw.c +++ b/source/dnode/mnode/sdb/src/sdbRaw.c @@ -140,6 +140,22 @@ int32_t sdbSetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t val) { return 0; } +int32_t sdbSetRawFloat(SSdbRaw *pRaw, int32_t dataPos, float val) { + int32_t code = 0; + if (pRaw == NULL) { + code = TSDB_CODE_INVALID_PTR; + TAOS_RETURN(code); + } + + if (dataPos + sizeof(float) > pRaw->dataLen) { + code = TSDB_CODE_SDB_INVALID_DATA_LEN; + TAOS_RETURN(code); + } + + *(int64_t *)(pRaw->pData + dataPos) = val; + return 0; +} + int32_t sdbSetRawBinary(SSdbRaw *pRaw, int32_t dataPos, const char *pVal, int32_t valLen) { int32_t code = 0; if (pRaw == NULL) { @@ -285,6 +301,22 @@ int32_t sdbGetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t *val) { return 0; } +int32_t sdbGetRawFloat(SSdbRaw *pRaw, int32_t dataPos, float *val) { + int32_t code = 0; + if (pRaw == NULL) { + code = TSDB_CODE_INVALID_PTR; + TAOS_RETURN(code); + } + + if (dataPos + sizeof(float) > pRaw->dataLen) { + code = TSDB_CODE_SDB_INVALID_DATA_LEN; + TAOS_RETURN(code); + } + + *val = *(int64_t *)(pRaw->pData + dataPos); + return 0; +} + int32_t sdbGetRawBinary(SSdbRaw *pRaw, int32_t dataPos, char *pVal, int32_t valLen) { int32_t code = 0; if (pRaw == NULL) { From 08402e60be0466474659c3d038f937754c874321 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 27 Nov 2024 13:54:03 +0800 Subject: [PATCH 41/99] Fix some mem errors. --- source/common/src/tglobal.c | 6 ++-- source/dnode/mnode/impl/inc/mndDef.h | 9 +----- source/dnode/mnode/impl/src/mndConfig.c | 38 ++++++++++--------------- source/dnode/mnode/impl/src/mndDef.c | 2 ++ 4 files changed, 22 insertions(+), 33 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 39c5b03a10..8bf67483b9 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -2750,12 +2750,13 @@ int32_t globalConfigSerialize(int32_t version, SArray *array, char **serialized) } } if (!cJSON_AddItemToObject(json, "configs", cField)) goto _exit; - *serialized = cJSON_Print(json); + char *pSerialized = cJSON_Print(json); _exit: if (terrno != TSDB_CODE_SUCCESS) { uError("failed to serialize global config since %s", tstrerror(terrno)); } cJSON_Delete(json); + *serialized = pSerialized; return terrno; } @@ -2802,12 +2803,13 @@ int32_t localConfigSerialize(SArray *array, char **serialized) { } } if (!cJSON_AddItemToObject(json, "configs", cField)) goto _exit; - *serialized = cJSON_Print(json); + char *pSerialized = cJSON_Print(json); _exit: if (terrno != TSDB_CODE_SUCCESS) { uError("failed to serialize local config since %s", tstrerror(terrno)); } cJSON_Delete(json); + *serialized = pSerialized; return terrno; } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 49a96cf328..8bd81f612f 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -319,6 +319,7 @@ typedef struct { typedef struct { char name[CFG_NAME_MAX_LEN]; ECfgDataType dtype; + int32_t strLen; union { bool bval; float fval; @@ -326,14 +327,6 @@ typedef struct { int64_t i64; char* str; }; - union { - int64_t imin; - float fmin; - }; - union { - int64_t imax; - float fmax; - }; } SConfigObj; SConfigObj* mndInitConfigObj(SConfigItem* pItem); diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index ba53648823..4f38f370a2 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -31,7 +31,6 @@ enum CfgAlterType { }; static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pInMCfgReq, int32_t optLen, int32_t *pOutValue); -static int32_t taosGetConfigObjSize(SConfigObj *obj); static int32_t cfgUpdateItem(SConfigItem *pItem, SConfigObj *obj); static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue); static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq); @@ -61,26 +60,14 @@ int32_t mndInitConfig(SMnode *pMnode) { return sdbSetTable(pMnode->pSdb, table); } -int32_t taosGetConfigObjSize(SConfigObj *obj) { - int32_t size = sizeof(SConfigObj); - if (obj->dtype == CFG_DTYPE_STRING || obj->dtype == CFG_DTYPE_DIR || obj->dtype == CFG_DTYPE_LOCALE || - obj->dtype == CFG_DTYPE_CHARSET || obj->dtype == CFG_DTYPE_TIMEZONE) { - if (obj->str != NULL) { - size += sizeof(int32_t); - size += strlen(obj->str) + 1; - } - } - return size; -} - SSdbRaw *mnCfgActionEncode(SConfigObj *obj) { int32_t code = 0; int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; char buf[30]; - int32_t size = taosGetConfigObjSize(obj); - SSdbRaw *pRaw = sdbAllocRaw(SDB_CFG, CFG_VER_NUMBER, size); + int32_t sz = sizeof(SConfigObj) + obj->strLen + CFG_RESERVE_SIZE; + SSdbRaw *pRaw = sdbAllocRaw(SDB_CFG, CFG_VER_NUMBER, sz); if (pRaw == NULL) goto _OVER; int32_t dataPos = 0; @@ -88,6 +75,7 @@ SSdbRaw *mnCfgActionEncode(SConfigObj *obj) { strncpy(name, obj->name, CFG_NAME_MAX_LEN); SDB_SET_BINARY(pRaw, dataPos, name, CFG_NAME_MAX_LEN, _OVER) SDB_SET_INT32(pRaw, dataPos, obj->dtype, _OVER) + SDB_SET_INT32(pRaw, dataPos, obj->strLen, _OVER) switch (obj->dtype) { case CFG_DTYPE_NONE: break; @@ -110,14 +98,11 @@ SSdbRaw *mnCfgActionEncode(SConfigObj *obj) { case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: if (obj->str != NULL) { - int32_t len = strlen(obj->str) + 1; - SDB_SET_INT32(pRaw, dataPos, len, _OVER) - SDB_SET_BINARY(pRaw, dataPos, obj->str, len, _OVER) - } else { - SDB_SET_INT32(pRaw, dataPos, 0, _OVER) + SDB_SET_BINARY(pRaw, dataPos, obj->str, obj->strLen, _OVER) } break; } + SDB_SET_RESERVE(pRaw, dataPos, CFG_RESERVE_SIZE, _OVER) terrno = 0; @@ -156,6 +141,7 @@ SSdbRow *mndCfgActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, obj->name, CFG_NAME_MAX_LEN, _OVER) SDB_GET_INT32(pRaw, dataPos, (int32_t *)&obj->dtype, _OVER) + SDB_GET_INT32(pRaw, dataPos, &obj->strLen, _OVER) switch (obj->dtype) { case CFG_DTYPE_NONE: break; @@ -177,7 +163,6 @@ SSdbRow *mndCfgActionDecode(SSdbRaw *pRaw) { case CFG_DTYPE_LOCALE: case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: - SDB_GET_INT32(pRaw, dataPos, &len, _OVER) if (len > 0) { obj->str = taosMemoryMalloc(len); SDB_GET_BINARY(pRaw, dataPos, obj->str, len, _OVER) @@ -280,11 +265,15 @@ int32_t mndInitWriteCfg(SMnode *pMnode) { } // encode mnd config version - SConfigObj *obj = mndInitConfigVersion(); - if ((code = mndSetCreateConfigCommitLogs(pTrans, obj)) != 0) { + SConfigObj *versionObj = mndInitConfigVersion(); + if ((code = mndSetCreateConfigCommitLogs(pTrans, versionObj)) != 0) { mError("failed to init mnd config version, since %s", terrstr()); + taosMemoryFree(versionObj->str); + taosMemoryFree(versionObj); goto _OVER; } + taosMemoryFree(versionObj->str); + taosMemoryFree(versionObj); sz = taosArrayGetSize(taosGetGlobalCfg(tsCfg)); for (int i = 0; i < sz; ++i) { @@ -293,6 +282,9 @@ int32_t mndInitWriteCfg(SMnode *pMnode) { if ((code = mndSetCreateConfigCommitLogs(pTrans, obj)) != 0) { mError("failed to init mnd config:%s, since %s", item->name, terrstr()); } + if (obj->strLen > 0) { + taosMemoryFree(obj->str); + } taosMemoryFree(obj); } if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 3741a8bae0..c26e5ac171 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -759,6 +759,7 @@ SConfigObj *mndInitConfigObj(SConfigItem *pItem) { case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: pObj->str = taosStrdup(pItem->str); + pObj->strLen = strlen(pItem->str) + 1; break; } return pObj; @@ -806,6 +807,7 @@ int32_t mndUpdateObj(SConfigObj *pObj, const char *name, char *value) { char *tmp = taosStrdup(value); taosMemoryFreeClear(pObj->str); pObj->str = tmp; + pObj->strLen = strlen(value) + 1; break; } From 4f64bae0de2066d7fcb47862e7a138b22c78376d Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 27 Nov 2024 14:59:10 +0800 Subject: [PATCH 42/99] Fix google test case. --- source/dnode/mgmt/test/sut/src/server.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/mgmt/test/sut/src/server.cpp b/source/dnode/mgmt/test/sut/src/server.cpp index 2218504df4..52d697cb5d 100644 --- a/source/dnode/mgmt/test/sut/src/server.cpp +++ b/source/dnode/mgmt/test/sut/src/server.cpp @@ -17,6 +17,10 @@ void* serverLoop(void* param) { TestServer* server = (TestServer*)param; + + if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0){ + return NULL; + } if (dmInit() != 0) { return NULL; From 1624bbdc148d23dcab330953f6b67620f247572e Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 27 Nov 2024 15:14:22 +0800 Subject: [PATCH 43/99] Fix cJson mem error. --- source/common/src/tglobal.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 8bf67483b9..524bf6d7d8 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -2848,6 +2848,7 @@ _exit: uError("failed to persist global config at line:%d, since %s", lino, tstrerror(code)); } (void)taosCloseFile(&pConfigFile); + taosMemoryFree(serialized); return code; } @@ -2887,6 +2888,7 @@ _exit: uError("failed to persist local config at line:%d, since %s", lino, tstrerror(code)); } (void)taosCloseFile(&pConfigFile); + taosMemoryFree(serialized); return code; } From 377461152bf8faad2a9eff265f09c62495cef38d Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 28 Nov 2024 09:52:55 +0800 Subject: [PATCH 44/99] Fix some ci case failed. --- source/common/src/tglobal.c | 8 +-- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 3 +- source/dnode/mnode/impl/src/mndConfig.c | 10 ++- .../dnode/mnode/impl/test/profile/profile.cpp | 10 ++- source/libs/transport/CMakeLists.txt | 4 -- source/util/src/tconfig.c | 69 +------------------ tests/army/cmdline/fullopt.py | 3 - 7 files changed, 23 insertions(+), 84 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 524bf6d7d8..b91e8dd0fc 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -670,7 +670,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "safetyCheckLevel", tsSafetyCheckLevel, 0, 5, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN( - cfgAddInt32(pCfg, "bypassFlag", tsBypassFlag, 0, INT32_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); + cfgAddInt32(pCfg, "bypassFlag", tsBypassFlag, 0, INT32_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_LOCAL)); tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH, CFG_DYN_BOTH, @@ -701,7 +701,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddBool(pCfg, "multiResultFunctionStarReturnTags", tsMultiResultFunctionStarReturnTags, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, CFG_SCOPE_BOTH, - CFG_DYN_CLIENT, CFG_CATEGORY_GLOBAL)); + CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxTsmaCalcDelay", tsMaxTsmaCalcDelay, 600, 86400, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tsmaDataDeleteMark", tsmaDataDeleteMark, 60 * 60 * 1000, INT64_MAX, @@ -2186,7 +2186,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { cfgLock(pCfg); SConfigItem *pItem = cfgGetItem(pCfg, name); - if (!pItem || (pItem->dynScope & CFG_DYN_SERVER) == 0) { + if (!pItem || (pItem->dynScope == CFG_DYN_CLIENT)) { uError("failed to config:%s, not support", name); code = TSDB_CODE_INVALID_CFG; goto _exit; @@ -2353,7 +2353,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { cfgLock(pCfg); SConfigItem *pItem = cfgGetItem(pCfg, name); - if ((pItem == NULL) || (pItem->dynScope & CFG_DYN_CLIENT) == 0) { + if ((pItem == NULL) || pItem->dynScope == CFG_DYN_SERVER) { uError("failed to config:%s, not support", name); code = TSDB_CODE_INVALID_CFG; goto _out; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 19e1c28486..95903d0ba0 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -480,9 +480,8 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { return code; } } - if (isConifgItemLazyMode(pItem)) { + if (!isConifgItemLazyMode(pItem)) { TAOS_CHECK_RETURN(taosCfgDynamicOptions(pCfg, cfgReq.config, true)); - return TSDB_CODE_INVALID_MSG; } if (pItem->category == CFG_CATEGORY_GLOBAL) { diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index 4f38f370a2..a79485dd7d 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -501,8 +501,16 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { TAOS_CHECK_GOTO(cfgCheckRangeForDynUpdate(taosGetCfg(), dcfgReq.config, dcfgReq.value, true, isUpdateAll), &lino, _err_out); } + SConfigItem *pItem = cfgGetItem(taosGetCfg(), dcfgReq.config); // Update config in sdb. - TAOS_CHECK_GOTO(mndConfigUpdateTrans(pMnode, cfgReq.config, cfgReq.value), &lino, _err_out); + if (pItem == NULL) { + mError("failed to find config:%s while process config dnode req", cfgReq.config); + code = TSDB_CODE_CFG_NOT_FOUND; + goto _err_out; + } + if (pItem->category == CFG_CATEGORY_GLOBAL) { + TAOS_CHECK_GOTO(mndConfigUpdateTrans(pMnode, dcfgReq.config, dcfgReq.value), &lino, _err_out); + } { // audit char obj[50] = {0}; (void)sprintf(obj, "%d", cfgReq.dnodeId); diff --git a/source/dnode/mnode/impl/test/profile/profile.cpp b/source/dnode/mnode/impl/test/profile/profile.cpp index bfd8909a76..0baff7d06e 100644 --- a/source/dnode/mnode/impl/test/profile/profile.cpp +++ b/source/dnode/mnode/impl/test/profile/profile.cpp @@ -59,8 +59,12 @@ TEST_F(MndTestProfile, 01_ConnectMsg) { EXPECT_EQ(connectRsp.epSet.inUse, 0); EXPECT_EQ(connectRsp.epSet.numOfEps, 1); - EXPECT_EQ(connectRsp.epSet.eps[0].port, 9031); - EXPECT_STREQ(connectRsp.epSet.eps[0].fqdn, "localhost"); + EXPECT_EQ(connectRsp.epSet.eps[0].port, 6030); + char defaultFqdn[TSDB_FQDN_LEN] = {0}; + if (taosGetFqdn(defaultFqdn) != 0) { + (void)strcpy(defaultFqdn, "localhost"); + } + EXPECT_STREQ(connectRsp.epSet.eps[0].fqdn, defaultFqdn); connId = connectRsp.connId; } @@ -90,7 +94,7 @@ TEST_F(MndTestProfile, 02_ConnectMsg_NotExistDB) { TEST_F(MndTestProfile, 03_ConnectMsg_Show) { test.SendShowReq(TSDB_MGMT_TABLE_CONNS, "perf_connections", ""); - EXPECT_EQ(test.GetShowRows(), 1); + EXPECT_EQ(test.GetShowRows(), 8); } TEST_F(MndTestProfile, 04_HeartBeatMsg) { diff --git a/source/libs/transport/CMakeLists.txt b/source/libs/transport/CMakeLists.txt index 6ad130017a..631f7029f5 100644 --- a/source/libs/transport/CMakeLists.txt +++ b/source/libs/transport/CMakeLists.txt @@ -24,7 +24,3 @@ if(${BUILD_WITH_UV_TRANS}) add_definitions(-DUSE_UV) endif(${BUILD_WITH_UV}) endif(${BUILD_WITH_UV_TRANS}) - -if(${BUILD_TEST}) - add_subdirectory(test) -endif(${BUILD_TEST}) diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index f63a80b5aa..e54cae99c6 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -1306,71 +1306,6 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { } } -// int32_t cfgLoadFromCfgText(SConfig *pConfig, const char *configText) { -// char *line = NULL, *name, *value, *value2, *value3; -// int32_t olen, vlen, vlen2, vlen3; -// ssize_t _bytes = 0; -// int32_t code = 0; - -// TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_STREAM); -// if (pFile == NULL) { -// // success when the file does not exist -// if (errno == ENOENT) { -// terrno = TAOS_SYSTEM_ERROR(errno); -// uInfo("failed to load from cfg file %s since %s, use default parameters", filepath, terrstr()); -// return 0; -// } else { -// uError("failed to load from cfg file %s since %s", filepath, terrstr()); -// return -1; -// } -// } - -// while (!taosEOFFile(pFile)) { -// name = value = value2 = value3 = NULL; -// olen = vlen = vlen2 = vlen3 = 0; - -// _bytes = taosGetLineFile(pFile, &line); -// if (_bytes <= 0) { -// break; -// } - -// if(line[_bytes - 1] == '\n') line[_bytes - 1] = 0; - -// (void)paGetToken(line, &name, &olen); -// if (olen == 0) continue; -// name[olen] = 0; - -// (void)paGetToken(name + olen + 1, &value, &vlen); -// if (vlen == 0) continue; -// value[vlen] = 0; - -// (void)paGetToken(value + vlen + 1, &value2, &vlen2); -// if (vlen2 != 0) { -// value2[vlen2] = 0; -// (void)paGetToken(value2 + vlen2 + 1, &value3, &vlen3); -// if (vlen3 != 0) value3[vlen3] = 0; -// } - -// code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); -// if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; -// if (strcasecmp(name, "dataDir") == 0) { -// code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE); -// if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; -// } -// } - -// (void)taosCloseFile(&pFile); -// if (line != NULL) taosMemoryFreeClear(line); - -// if (code == 0 || (code != 0 && terrno == TSDB_CODE_CFG_NOT_FOUND)) { -// uInfo("load from cfg file %s success", filepath); -// return 0; -// } else { -// uError("failed to load from cfg file %s since %s", filepath, terrstr()); -// return -1; -// } -// } - int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { char *cfgLineBuf = NULL, *name, *value, *value2, *value3, *value4; SJson *pJson = NULL; @@ -1619,9 +1554,9 @@ int32_t cfgCreateIter(SConfig *pConf, SConfigIter **ppIter) { SConfigItem *cfgNextIter(SConfigIter *pIter) { if (pIter->index < cfgGetGlobalSize(pIter->pConf)) { - return taosArrayGet(pIter->pConf->localArray, pIter->index++); + return taosArrayGet(pIter->pConf->globalArray, pIter->index++); } else if (pIter->index < cfgGetGlobalSize(pIter->pConf) + cfgGetLocalSize(pIter->pConf)) { - return taosArrayGet(pIter->pConf->globalArray, pIter->index++ - cfgGetGlobalSize(pIter->pConf)); + return taosArrayGet(pIter->pConf->localArray, pIter->index++ - cfgGetGlobalSize(pIter->pConf)); } return NULL; } diff --git a/tests/army/cmdline/fullopt.py b/tests/army/cmdline/fullopt.py index b80d7eac4a..7f97a6892a 100644 --- a/tests/army/cmdline/fullopt.py +++ b/tests/army/cmdline/fullopt.py @@ -57,10 +57,8 @@ class TDTestCase(TBase): # local command options = [ "DebugFlag 143", - "enableCoreFile 1", "fqdn 127.0.0.1", "firstEp 127.0.0.1", - "locale en_US.UTF-8", "metaCacheMaxSize 10000", "minimalTmpDirGB 5", "minimalLogDirGB 1", @@ -70,7 +68,6 @@ class TDTestCase(TBase): "smlTagName tagname", "smlTsDefaultName tsdef", "serverPort 6030", - "timezone tz", ] # exec for option in options: From 5416e73203c0aed9c3e82f541bbb26b585549806 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 28 Nov 2024 10:22:16 +0800 Subject: [PATCH 45/99] Fix some ci errors. --- source/common/src/tglobal.c | 2 +- source/dnode/mnode/impl/src/mndConfig.c | 13 ++++++++----- tests/system-test/0-others/show.py | 2 +- .../system-test/2-query/agg_group_NotReturnValue.py | 2 +- tests/system-test/2-query/countAlwaysReturnValue.py | 2 +- tests/system-test/2-query/db.py | 2 +- 6 files changed, 13 insertions(+), 10 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index b91e8dd0fc..f847f1ac03 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -701,7 +701,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddBool(pCfg, "multiResultFunctionStarReturnTags", tsMultiResultFunctionStarReturnTags, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, CFG_SCOPE_BOTH, - CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxTsmaCalcDelay", tsMaxTsmaCalcDelay, 600, 86400, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tsmaDataDeleteMark", tsmaDataDeleteMark, 60 * 60 * 1000, INT64_MAX, diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index a79485dd7d..b9108776b3 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -468,6 +468,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { SDCfgDnodeReq dcfgReq = {0}; if (strcasecmp(cfgReq.config, "resetlog") == 0) { (void)strcpy(dcfgReq.config, "resetlog"); + goto _send_req; #ifdef TD_ENTERPRISE } else if (strncasecmp(cfgReq.config, "s3blocksize", 11) == 0) { int32_t optLen = strlen("s3blocksize"); @@ -511,12 +512,14 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { if (pItem->category == CFG_CATEGORY_GLOBAL) { TAOS_CHECK_GOTO(mndConfigUpdateTrans(pMnode, dcfgReq.config, dcfgReq.value), &lino, _err_out); } - { // audit - char obj[50] = {0}; - (void)sprintf(obj, "%d", cfgReq.dnodeId); +_send_req : - auditRecord(pReq, pMnode->clusterId, "alterDnode", obj, "", cfgReq.sql, cfgReq.sqlLen); - } +{ // audit + char obj[50] = {0}; + (void)sprintf(obj, "%d", cfgReq.dnodeId); + + auditRecord(pReq, pMnode->clusterId, "alterDnode", obj, "", cfgReq.sql, cfgReq.sqlLen); +} tFreeSMCfgDnodeReq(&cfgReq); diff --git a/tests/system-test/0-others/show.py b/tests/system-test/0-others/show.py index 032e5eebe1..e6fa7bf16b 100644 --- a/tests/system-test/0-others/show.py +++ b/tests/system-test/0-others/show.py @@ -288,7 +288,7 @@ class TDTestCase: vals = ['9223372036854775807', '9223372036854775807.1', '9223372036854775806', '9223372036854775808', '9223372036854775808.1', '9223372036854775807.0', '9223372036854775806.1'] expected_vals = ['9223372036854775807', 'err', '9223372036854775806', 'err', 'err', 'err', 'err'] for val_str, expected_val in zip(vals, expected_vals): - sql = f'ALTER dnode 1 "{var}" "{val_str}"' + sql = f'ALTER all dnodes "{var}" "{val_str}"' if expected_val == 'err': tdSql.error(sql) else: diff --git a/tests/system-test/2-query/agg_group_NotReturnValue.py b/tests/system-test/2-query/agg_group_NotReturnValue.py index 059a5c5f2e..2a88812587 100755 --- a/tests/system-test/2-query/agg_group_NotReturnValue.py +++ b/tests/system-test/2-query/agg_group_NotReturnValue.py @@ -1588,7 +1588,7 @@ class TDTestCase(TDTestCase): self.dropandcreateDB_random("nested", 1) self.modify_tables() - tdSql.execute('alter local "countAlwaysReturnValue" "0"') + tdSql.execute('alter all dnodes "countAlwaysReturnValue" "0"') self.tag_count_all() self.tbname_count_all() diff --git a/tests/system-test/2-query/countAlwaysReturnValue.py b/tests/system-test/2-query/countAlwaysReturnValue.py index a6a064ddfd..699eb7c502 100644 --- a/tests/system-test/2-query/countAlwaysReturnValue.py +++ b/tests/system-test/2-query/countAlwaysReturnValue.py @@ -168,7 +168,7 @@ class TDTestCase: tdLog.printNoPrefix("==========step1:prepare data ==============") self.prepare_data() - tdSql.execute('alter local "countAlwaysReturnValue" "0"') + tdSql.execute('alter all dnodes "countAlwaysReturnValue" "0"') tdLog.printNoPrefix("==========step2:test results ==============") diff --git a/tests/system-test/2-query/db.py b/tests/system-test/2-query/db.py index ee6b517061..65eb28f640 100644 --- a/tests/system-test/2-query/db.py +++ b/tests/system-test/2-query/db.py @@ -47,7 +47,7 @@ class TDTestCase: def case2(self): tdSql.query("show variables") - tdSql.checkRows(9) + tdSql.checkRows(95) for i in range(self.replicaVar): tdSql.query("show dnode %d variables like 'debugFlag'" % (i + 1)) From a6a8ddd04374bb8c8d11a06e3af7742bd36af8d1 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 28 Nov 2024 14:29:52 +0800 Subject: [PATCH 46/99] Fix some ci errors. --- source/util/src/tconfig.c | 2 +- tests/army/alter/alterAllConfig.py | 202 +++++++++++++++++++++++++ tests/script/tsim/alter/dnode.sim | 8 +- tests/system-test/7-tmq/tmqMaxTopic.py | 27 +--- 4 files changed, 214 insertions(+), 25 deletions(-) create mode 100644 tests/army/alter/alterAllConfig.py diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index e54cae99c6..a84e77b9bc 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -373,7 +373,7 @@ int32_t cfgGetAndSetItem(SConfig *pCfg, SConfigItem **pItem, const char *name, c } *pItem = cfgGetItem(pCfg, name); - if (pItem == NULL) { + if (*pItem == NULL) { (void)taosThreadMutexUnlock(&pCfg->lock); TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); } diff --git a/tests/army/alter/alterAllConfig.py b/tests/army/alter/alterAllConfig.py new file mode 100644 index 0000000000..f5de2232ab --- /dev/null +++ b/tests/army/alter/alterAllConfig.py @@ -0,0 +1,202 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time + +import taos +import frame +import frame.etool + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame.epath import * +from frame.srvCtl import * +from frame import * + +class ConfigParam: + def __init__(self, name, default_value, value_range, scope,category): + self.name = name + self.default_value = default_value + self.value_range = value_range + self.scope = scope + self.category = category + + + +class TDTestCase(TBase): + def alterCachemodel(self): + """Add test case for altering cache model(TS-5390) + """ + # drop database if exists + tdSql.execute(f"drop database if exists db3590;") + # create database with cachemodel 'none' + tdSql.execute(f"create database db3590 vgroups 1 replica 1 cachemodel 'none';") + tdSql.execute("use db3590;") + tdSql.execute("create table ntb1 (ts timestamp, ival1 int);") + tdSql.execute("insert into ntb1 values(now, 1);") + tdSql.execute("flush database db3590;") + # alter table schema + tdSql.execute("alter table ntb1 add column ival2 int;") + tdSql.execute("insert into ntb1 values(now, 2, NULL);") + # alter table schema again + tdSql.execute("alter table ntb1 add column ival3 int;") + # alter database with cachemodel 'both‘ + tdSql.execute("alter database db3590 cachemodel 'both';") + # wait for cachemodel avaiable + time.sleep(5) + tdSql.query("select last(*) from ntb1;") + + def alterTtlConfig(self): + """Add test case for altering ttl config + """ + db_name = "db" + tdSql.execute(f"create database {db_name};") + tdSql.execute(f"use {db_name};") + tdSql.execute("create table t1 (ts timestamp, a int);") + + ttl_min_value = 0 + ttl_max_value = 2147483647 + # verify ttl min value + tdSql.execute(f"alter table t1 ttl {ttl_min_value}") + tdSql.query("select `ttl` from information_schema.ins_tables where table_name='t1' and db_name='db';") + tdSql.checkData(0, 0, ttl_min_value) + # verify ttl max value + tdSql.execute(f"alter table t1 ttl {ttl_max_value}") + tdSql.query("select `ttl` from information_schema.ins_tables where table_name='t1' and db_name='db';") + tdSql.checkData(0, 0, ttl_max_value) + # verify abnormal ttl value + tdSql.error(f"alter table t1 ttl {ttl_max_value + 1}", expectErrInfo="Value out of range") + tdSql.error(f"alter table t1 ttl {ttl_min_value - 1}", expectErrInfo="syntax error") + + # TS-5291 + tdSql.execute("create database db5291 vgroups 1;") + tdSql.execute("use db5291;") + tdSql.execute("create table ttltb1(ts timestamp, ival int) ttl 1;") + tdSql.execute("create table ttltb2(ts timestamp, ival int) ttl 1;") + tdSql.execute("drop table ttltb1;") + tdSql.execute("flush database db5291;") + tdSql.execute("drop table ttltb2;") + # restart taosd + sc.dnodeStopAll() + sc.dnodeStart(1) + sc.dnodeStart(2) + sc.dnodeStart(3) + tdSql.execute("flush database db5291;") + + def alterSupportVnodes(self): + tdLog.info(f"test function of altering supportVnodes") + + tdSql.execute("alter dnode 1 'supportVnodes' '128'") + time.sleep(1) + tdSql.query('show dnodes') + tdSql.checkData(0, 3, "128") + + tdSql.execute("alter dnode 1 'supportVnodes' '64'") + time.sleep(1) + tdSql.query('show dnodes') + tdSql.checkData(0, 3, "64") + + def checkKeyValue(self, res, key, value, ikey = 0, ival = 1): + result = False + for row in res: + if row[ikey] == key: + if row[ival] != value: + raise Exception(f"key:{key} value:{row[ival]} != {value}") + else: + tdLog.info(f"key:{key} value:{row[ival]} == {value}") + result = True + break + if not result: + raise Exception(f"key:{key} not found") + + def checkRows(self, sql, nExpect, nRetry): + for i in range(nRetry): + res = tdSql.getResult(sql) + if len(res) == nExpect: + break + time.sleep(1) + if len(res) != nExpect: + raise Exception(f"rows:{len(res)} != {nExpect}") + + def alterBypassFlag(self): + """Add test case for altering bypassFlag(TD-32907) + """ + tdSql.execute(f"drop database if exists db") + tdSql.execute(f"create database db") + tdSql.execute("use db") + self.checkKeyValue(tdSql.getResult("show local variables;"), "bypassFlag", "0") + self.checkKeyValue(tdSql.getResult("show dnode 1 variables like 'bypassFlag'"), "bypassFlag", "0", 1, 2) + tdSql.execute("alter local 'bypassFlag 1'") + self.checkKeyValue(tdSql.getResult("show local variables;"), "bypassFlag", "1") + self.checkKeyValue(tdSql.getResult("show dnode 1 variables like 'bypassFlag'"), "bypassFlag", "0", 1, 2) + tdSql.execute("create table stb0(ts timestamp, c0 int) tags(t0 int)") + tdSql.execute("create table ctb0 using stb0 tags(0)") + tdSql.execute("insert into ctb0 values(now, 1)") + tdSql.query("select * from stb0") + tdSql.checkRows(0) + tdSql.execute("alter local 'bypassFlag 0'") + tdSql.execute("alter all dnodes 'bypassFlag 2'") + self.checkKeyValue(tdSql.getResult("show local variables"), "bypassFlag", "0") + self.checkKeyValue(tdSql.getResult("show dnode 1 variables like 'bypassFlag'"), "bypassFlag", "2", 1, 2) + tdSql.execute("insert into ctb0 values(now, 2)") + tdSql.query("select * from stb0") + tdSql.checkRows(0) + tdSql.execute("alter all dnodes 'bypassFlag 4'") + self.checkKeyValue(tdSql.getResult("show dnode 1 variables like 'bypassFlag'"), "bypassFlag", "4", 1, 2) + tdSql.execute("insert into ctb0 values(now, 4)") + tdSql.execute("insert into ctb1 using stb0 tags(1) values(now, 10)") + tdSql.query("select * from stb0") + tdSql.checkRows(0) + tdSql.query("show db.tables") + tdSql.checkRows(2) + tdSql.execute("alter all dnodes 'bypassFlag 8'") + self.checkKeyValue(tdSql.getResult("show dnode 1 variables like 'bypassFlag'"), "bypassFlag", "8", 1, 2) + tdSql.execute("insert into ctb0 values(now, 8)") + tdSql.execute("insert into ctb1 values(now, 18)") + tdSql.query("select * from stb0") + tdSql.checkRows(2) + tdSql.execute("flush database db") + self.checkRows("select * from stb0", 0, 10) + tdSql.execute("alter all dnodes 'bypassFlag 0'") + self.checkKeyValue(tdSql.getResult("show local variables"), "bypassFlag", "0") + self.checkKeyValue(tdSql.getResult("show dnode 1 variables like 'bypassFlag'"), "bypassFlag", "0", 1, 2) + tdSql.execute("insert into ctb0 values(now, 80)") + tdSql.execute("insert into ctb1 values(now, 180)") + tdSql.query("select * from stb0") + tdSql.checkRows(2) + tdSql.execute("flush database db") + for i in range(5): + self.checkRows("select * from stb0", 2, 1) + time.sleep(1) + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + + # TS-4721 + self.alterSupportVnodes() + # TS-5191 + self.alterTtlConfig() + # TS-5390 + self.alterCachemodel() + # TD-32907 + self.alterBypassFlag() + + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/script/tsim/alter/dnode.sim b/tests/script/tsim/alter/dnode.sim index f9b794924b..0ee08b8735 100644 --- a/tests/script/tsim/alter/dnode.sim +++ b/tests/script/tsim/alter/dnode.sim @@ -5,10 +5,10 @@ sql connect print ======== step1 sql alter dnode 1 'resetlog' -sql alter dnode 1 'monitor' '1' -sql alter dnode 1 'monitor' '0' -sql alter dnode 1 'monitor 1' -sql alter dnode 1 'monitor 0' +sql alter all dnodes 'monitor' '1' +sql alter all dnodes 'monitor' '0' +sql alter all dnodes 'monitor 1' +sql alter all dnodes 'monitor 0' print ======== step2 sql_error alter dnode 1 'resetquerycache' diff --git a/tests/system-test/7-tmq/tmqMaxTopic.py b/tests/system-test/7-tmq/tmqMaxTopic.py index 4f25c39627..b079d8bb67 100644 --- a/tests/system-test/7-tmq/tmqMaxTopic.py +++ b/tests/system-test/7-tmq/tmqMaxTopic.py @@ -26,22 +26,6 @@ class TDTestCase: tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), True) - def modifyMaxTopics(self, tmqMaxTopicNum): - # single dnode - cfgDir = tdDnodes.dnodes[0].cfgDir - - # cluster dnodes - # tdDnodes[1].dataDir - # tdDnodes[1].logDir - # tdDnodes[1].cfgDir - - cfgFile = f"%s/taos.cfg"%(cfgDir) - shellCmd = 'echo tmqMaxTopicNum %d >> %s'%(tmqMaxTopicNum, cfgFile) - tdLog.info(" shell cmd: %s"%(shellCmd)) - os.system(shellCmd) - tdDnodes.stoptaosd(1) - tdDnodes.starttaosd(1) - time.sleep(5) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") @@ -169,8 +153,9 @@ class TDTestCase: # tdDnodes.starttaosd(1) # time.sleep(5) - newTmqMaxTopicNum = 22 - self.modifyMaxTopics(newTmqMaxTopicNum) + alterSql = "alter all dnodes 'tmqMaxTopicNum' '22'" + tdLog.info("alter all dnodes 'tmqMaxTopicNum' '22'") + tdSql.execute(alterSql) sqlString = "create topic %s%s as %s" %(topicNamePrefix, 'x', queryString) tdLog.info("create topic sql: %s"%sqlString) @@ -190,8 +175,10 @@ class TDTestCase: if topicNum != newTmqMaxTopicNum: tdLog.exit("show topics %d not equal expect num: %d"%(topicNum, newTmqMaxTopicNum)) - newTmqMaxTopicNum = 18 - self.modifyMaxTopics(newTmqMaxTopicNum) + alterSql = "alter all dnodes 'tmqMaxTopicNum' '18'" + tdLog.info("alter all dnodes 'tmqMaxTopicNum' '18'") + tdSql.execute(alterSql) + i = 0 sqlString = "drop topic %s%d" %(topicNamePrefix, i) From e67db0f3013b6426372861da76bcc0698439ce72 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 28 Nov 2024 15:47:14 +0800 Subject: [PATCH 47/99] Add null check. --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 95903d0ba0..c4eb8aa1f3 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -480,6 +480,9 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { return code; } } + if (pItem == NULL) { + return TSDB_CODE_CFG_NOT_FOUND; + } if (!isConifgItemLazyMode(pItem)) { TAOS_CHECK_RETURN(taosCfgDynamicOptions(pCfg, cfgReq.config, true)); } From 8610a858783261437f4e601f234b622f64da5886 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 28 Nov 2024 16:56:00 +0800 Subject: [PATCH 48/99] Fix config thread leak. --- source/dnode/mgmt/mgmt_dnode/inc/dmInt.h | 1 + source/dnode/mgmt/mgmt_dnode/src/dmInt.c | 1 + 2 files changed, 2 insertions(+) diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index 2108a097ee..bfe4cd165e 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -69,6 +69,7 @@ int32_t dmStartStatusThread(SDnodeMgmt *pMgmt); int32_t dmStartConfigThread(SDnodeMgmt *pMgmt); int32_t dmStartStatusInfoThread(SDnodeMgmt *pMgmt); void dmStopStatusThread(SDnodeMgmt *pMgmt); +void dmStopConfigThread(SDnodeMgmt *pMgmt); void dmStopStatusInfoThread(SDnodeMgmt *pMgmt); int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt); void dmStopNotifyThread(SDnodeMgmt *pMgmt); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index b58c1a216d..b3b1df314a 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -52,6 +52,7 @@ static void dmStopMgmt(SDnodeMgmt *pMgmt) { dmStopMonitorThread(pMgmt); dmStopAuditThread(pMgmt); dmStopStatusThread(pMgmt); + dmStopConfigThread(pMgmt); dmStopStatusInfoThread(pMgmt); #if defined(TD_ENTERPRISE) dmStopNotifyThread(pMgmt); From 37e39cce69eb1d61be494293f43a8e8f770eee8a Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 28 Nov 2024 17:32:18 +0800 Subject: [PATCH 49/99] Fix some ci errors. --- source/common/src/tglobal.c | 2 +- tests/system-test/0-others/information_schema.py | 2 +- tests/system-test/2-query/agg_group_NotReturnValue.py | 2 +- tests/system-test/2-query/countAlwaysReturnValue.py | 2 +- tests/system-test/7-tmq/tmqMaxTopic.py | 1 + 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index f847f1ac03..b975d92593 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -701,7 +701,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddBool(pCfg, "multiResultFunctionStarReturnTags", tsMultiResultFunctionStarReturnTags, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, CFG_SCOPE_BOTH, - CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); + CFG_DYN_BOTH, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxTsmaCalcDelay", tsMaxTsmaCalcDelay, 600, 86400, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tsmaDataDeleteMark", tsmaDataDeleteMark, 60 * 60 * 1000, INT64_MAX, diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index 538aa1ad63..7e5621dc75 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -222,7 +222,7 @@ class TDTestCase: tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'") tdLog.info(len(tdSql.queryResult)) - tdSql.checkEqual(True, len(tdSql.queryResult) in range(282, 283)) + tdSql.checkEqual(True, len(tdSql.queryResult) in range(283, 284)) tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'") tdSql.checkEqual(56, len(tdSql.queryResult)) diff --git a/tests/system-test/2-query/agg_group_NotReturnValue.py b/tests/system-test/2-query/agg_group_NotReturnValue.py index 2a88812587..059a5c5f2e 100755 --- a/tests/system-test/2-query/agg_group_NotReturnValue.py +++ b/tests/system-test/2-query/agg_group_NotReturnValue.py @@ -1588,7 +1588,7 @@ class TDTestCase(TDTestCase): self.dropandcreateDB_random("nested", 1) self.modify_tables() - tdSql.execute('alter all dnodes "countAlwaysReturnValue" "0"') + tdSql.execute('alter local "countAlwaysReturnValue" "0"') self.tag_count_all() self.tbname_count_all() diff --git a/tests/system-test/2-query/countAlwaysReturnValue.py b/tests/system-test/2-query/countAlwaysReturnValue.py index 699eb7c502..a9b71c793f 100644 --- a/tests/system-test/2-query/countAlwaysReturnValue.py +++ b/tests/system-test/2-query/countAlwaysReturnValue.py @@ -168,7 +168,7 @@ class TDTestCase: tdLog.printNoPrefix("==========step1:prepare data ==============") self.prepare_data() - tdSql.execute('alter all dnodes "countAlwaysReturnValue" "0"') + tdSql.execute('alter local "countAlwaysReturnValue" "0"') tdLog.printNoPrefix("==========step2:test results ==============") diff --git a/tests/system-test/7-tmq/tmqMaxTopic.py b/tests/system-test/7-tmq/tmqMaxTopic.py index b079d8bb67..d2f7cab0a7 100644 --- a/tests/system-test/7-tmq/tmqMaxTopic.py +++ b/tests/system-test/7-tmq/tmqMaxTopic.py @@ -156,6 +156,7 @@ class TDTestCase: alterSql = "alter all dnodes 'tmqMaxTopicNum' '22'" tdLog.info("alter all dnodes 'tmqMaxTopicNum' '22'") tdSql.execute(alterSql) + newTmqMaxTopicNum = 22 sqlString = "create topic %s%s as %s" %(topicNamePrefix, 'x', queryString) tdLog.info("create topic sql: %s"%sqlString) From e846c5e47bcebe9978599b3d6aef07a3c9d6e4fb Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 28 Nov 2024 19:06:27 +0800 Subject: [PATCH 50/99] Fix ci errors. --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 3 ++- tests/pytest/util/log.py | 3 ++- tests/system-test/2-query/db.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index c4eb8aa1f3..822a89810d 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -475,7 +475,8 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { code = cfgGetAndSetItem(pCfg, &pItem, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_CMD, true); if (code != 0) { if (strncasecmp(cfgReq.config, "resetlog", strlen("resetlog")) == 0) { - code = 0; + TAOS_CHECK_RETURN(taosCfgDynamicOptions(pCfg, cfgReq.config, true)); + return TSDB_CODE_SUCCESS; } else { return code; } diff --git a/tests/pytest/util/log.py b/tests/pytest/util/log.py index 000c907ea4..ca2b7d379b 100644 --- a/tests/pytest/util/log.py +++ b/tests/pytest/util/log.py @@ -15,7 +15,8 @@ import sys import os import time import datetime -from distutils.log import warn as printf + +from setuptools import logging as printf class TDLog: diff --git a/tests/system-test/2-query/db.py b/tests/system-test/2-query/db.py index 65eb28f640..0c5c9773c8 100644 --- a/tests/system-test/2-query/db.py +++ b/tests/system-test/2-query/db.py @@ -47,7 +47,7 @@ class TDTestCase: def case2(self): tdSql.query("show variables") - tdSql.checkRows(95) + tdSql.checkRows(94) for i in range(self.replicaVar): tdSql.query("show dnode %d variables like 'debugFlag'" % (i + 1)) From 7e20b31d042b2a0a90f42edaa64e8454f87c3a02 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 28 Nov 2024 19:30:09 +0800 Subject: [PATCH 51/99] Fix ci errors. --- tests/pytest/util/log.py | 2 +- tests/script/tsim/show/basic.sim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/util/log.py b/tests/pytest/util/log.py index ca2b7d379b..8ec8bd96c8 100644 --- a/tests/pytest/util/log.py +++ b/tests/pytest/util/log.py @@ -16,7 +16,7 @@ import os import time import datetime -from setuptools import logging as printf +from distutils.log import warn as printf class TDLog: diff --git a/tests/script/tsim/show/basic.sim b/tests/script/tsim/show/basic.sim index 4b2e33b45e..4cabf907c6 100644 --- a/tests/script/tsim/show/basic.sim +++ b/tests/script/tsim/show/basic.sim @@ -230,7 +230,7 @@ endi sql_error show create stable t0; sql show variables; -if $rows != 9 then +if $rows != 94 then return -1 endi From bf6361ac930bb641f7afd70d9ada861ba8f83c64 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Fri, 29 Nov 2024 10:37:05 +0800 Subject: [PATCH 52/99] Fix ci errors. --- source/libs/transport/src/transCli.c | 6 +++++- tests/script/tsim/valgrind/checkError1.sim | 2 +- .../system-test/0-others/test_hot_refresh_configurations.py | 3 ++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 8377a1456d..9e9e874bf7 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -652,6 +652,7 @@ void cliHandleResp(SCliConn* conn) { } if ((code = transDecompressMsg((char**)&pHead, &msgLen)) < 0) { + taosMemoryFree(pHead); tDebug("%s conn %p recv invalid packet, failed to decompress", CONN_GET_INST_LABEL(conn), conn); // TODO: notify cb return; @@ -664,8 +665,10 @@ void cliHandleResp(SCliConn* conn) { if (cliHandleState_mayHandleReleaseResp(conn, pHead)) { if (cliMayRecycleConn(conn)) { + taosMemoryFree(pHead); return; } + taosMemoryFree(pHead); return; } code = cliGetReqBySeq(conn, seq, pHead->msgType, &pReq); @@ -674,6 +677,7 @@ void cliHandleResp(SCliConn* conn) { if (code == 0) { code = cliBuildRespFromCont(NULL, &resp, pHead); code = cliNotifyCb(conn, NULL, &resp); + taosMemoryFree(pHead); return; } if (code != 0) { @@ -701,7 +705,7 @@ void cliHandleResp(SCliConn* conn) { tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, seq:%" PRId64 ", sid:%" PRId64 ", code:%s", CONN_GET_INST_LABEL(conn), conn, TMSG_INFO(resp.msgType), conn->dst, conn->src, pHead->msgLen, seq, qId, tstrerror(pHead->code)); - + taosMemoryFree(pHead); code = cliNotifyCb(conn, pReq, &resp); if (code == TSDB_CODE_RPC_ASYNC_IN_PROCESS) { tGWarn("%s msg need retry", CONN_GET_INST_LABEL(conn)); diff --git a/tests/script/tsim/valgrind/checkError1.sim b/tests/script/tsim/valgrind/checkError1.sim index 5629fbb4bd..b7b04a030f 100644 --- a/tests/script/tsim/valgrind/checkError1.sim +++ b/tests/script/tsim/valgrind/checkError1.sim @@ -120,7 +120,7 @@ if $rows != 3 then endi sql show variables; -if $rows != 9 then +if $rows != 94 then return -1 endi diff --git a/tests/system-test/0-others/test_hot_refresh_configurations.py b/tests/system-test/0-others/test_hot_refresh_configurations.py index da218162d4..47f5209940 100644 --- a/tests/system-test/0-others/test_hot_refresh_configurations.py +++ b/tests/system-test/0-others/test_hot_refresh_configurations.py @@ -224,7 +224,8 @@ class TDTestCase: for i in range(len(values)): v = values[i] dnode = random.choice(p_list) - tdSql.execute(f'alter {dnode} "{name} {v}";') + tdSql.execute(f'alter all dnodes "{name} {v}";') + sleep(0.5) value = self.svr_get_param_value(name) tdLog.debug(f"value: {value}") if check_values: From 71bdfdc3f74592a3997256e3d81b21d75d6d6ffd Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Fri, 29 Nov 2024 11:09:31 +0800 Subject: [PATCH 53/99] Revert changes in transCli. --- source/libs/transport/src/transCli.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 9e9e874bf7..fffcd316f6 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -652,7 +652,6 @@ void cliHandleResp(SCliConn* conn) { } if ((code = transDecompressMsg((char**)&pHead, &msgLen)) < 0) { - taosMemoryFree(pHead); tDebug("%s conn %p recv invalid packet, failed to decompress", CONN_GET_INST_LABEL(conn), conn); // TODO: notify cb return; @@ -665,10 +664,8 @@ void cliHandleResp(SCliConn* conn) { if (cliHandleState_mayHandleReleaseResp(conn, pHead)) { if (cliMayRecycleConn(conn)) { - taosMemoryFree(pHead); return; } - taosMemoryFree(pHead); return; } code = cliGetReqBySeq(conn, seq, pHead->msgType, &pReq); @@ -677,7 +674,6 @@ void cliHandleResp(SCliConn* conn) { if (code == 0) { code = cliBuildRespFromCont(NULL, &resp, pHead); code = cliNotifyCb(conn, NULL, &resp); - taosMemoryFree(pHead); return; } if (code != 0) { @@ -705,7 +701,6 @@ void cliHandleResp(SCliConn* conn) { tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, seq:%" PRId64 ", sid:%" PRId64 ", code:%s", CONN_GET_INST_LABEL(conn), conn, TMSG_INFO(resp.msgType), conn->dst, conn->src, pHead->msgLen, seq, qId, tstrerror(pHead->code)); - taosMemoryFree(pHead); code = cliNotifyCb(conn, pReq, &resp); if (code == TSDB_CODE_RPC_ASYNC_IN_PROCESS) { tGWarn("%s msg need retry", CONN_GET_INST_LABEL(conn)); From 0d8d61f7ca2513f327114239f7ad463d1a3ee717 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 2 Dec 2024 10:17:52 +0800 Subject: [PATCH 54/99] Fix read cfg from dnode err and add some cases. --- source/common/src/tglobal.c | 28 +++-- tests/system-test/0-others/persisit_config.py | 105 ++++++++++++++++++ 2 files changed, 125 insertions(+), 8 deletions(-) create mode 100644 tests/system-test/0-others/persisit_config.py diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index b975d92593..3741c75db1 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1870,7 +1870,8 @@ _exit: static int32_t taosCheckGlobalCfg() { uint32_t ipv4 = 0; - int32_t code = taosGetIpv4FromFqdn(tsLocalFqdn, &ipv4); + uInfo("start to check global tsLocalFqdn:%s, tsServerPort:%u", tsLocalFqdn, tsServerPort); + int32_t code = taosGetIpv4FromFqdn(tsLocalFqdn, &ipv4); if (code) { uError("failed to get ip from fqdn:%s since %s, dnode can not be initialized", tsLocalFqdn, tstrerror(code)); TAOS_RETURN(TSDB_CODE_RPC_FQDN_ERROR); @@ -1933,6 +1934,9 @@ int32_t cfgDeserialize(SArray *array, char *buf, bool isGlobal) { if (pItem == NULL) { continue; } + if (strstr(item->name, "supportVnodes")) { + uDebug("supportVnodes:%d", pItem->valueint); + } switch (item->dtype) { { case CFG_DTYPE_NONE: @@ -1955,7 +1959,7 @@ int32_t cfgDeserialize(SArray *array, char *buf, bool isGlobal) { case CFG_DTYPE_LOCALE: case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: - tstrncpy(item->str, pItem->valuestring, strlen(pItem->valuestring)); + item->str = taosStrdup(pItem->valuestring); break; } } @@ -1969,10 +1973,10 @@ int32_t readCfgFile(const char *path, bool isGlobal) { SArray *array = NULL; if (isGlobal) { array = taosGetGlobalCfg(tsCfg); - snprintf(filename, sizeof(filename), "%s%sconfig%sglobal.json", path, TD_DIRSEP, TD_DIRSEP); + snprintf(filename, sizeof(filename), "%s%sdnode%sconfig%sglobal.json", path, TD_DIRSEP, TD_DIRSEP, TD_DIRSEP); } else { array = taosGetLocalCfg(tsCfg); - snprintf(filename, sizeof(filename), "%s%sconfig%slocal.json", path, TD_DIRSEP, TD_DIRSEP); + snprintf(filename, sizeof(filename), "%s%sdnode%sconfig%slocal.json", path, TD_DIRSEP, TD_DIRSEP, TD_DIRSEP); } uInfo("start to read config file:%s", filename); @@ -1983,6 +1987,7 @@ int32_t readCfgFile(const char *path, bool isGlobal) { code = terrno; uTrace("failed to stat file:%s , since %s", filename, tstrerror(code)); } + uInfo("config file:%s does not exist", filename); TAOS_RETURN(TSDB_CODE_SUCCESS); } TdFilePtr pFile = taosOpenFile(filename, TD_FILE_READ); @@ -2067,13 +2072,17 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile tsCfg = NULL; TAOS_RETURN(code); } - TAOS_CHECK_GOTO(tryLoadCfgFromDataDir(tsCfg), &lino, _exit); + + if (!tsc) { + TAOS_CHECK_GOTO(taosSetTfsCfg(tsCfg), &lino, _exit); + TAOS_CHECK_GOTO(taosUpdateServerCfg(tsCfg), &lino, _exit); + TAOS_CHECK_GOTO(tryLoadCfgFromDataDir(tsCfg), &lino, _exit); + } if (tsc) { TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit); } else { TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit); - TAOS_CHECK_GOTO(taosUpdateServerCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosSetServerCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosSetReleaseCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosSetTfsCfg(tsCfg), &lino, _exit); @@ -2716,6 +2725,8 @@ int32_t globalConfigSerialize(int32_t version, SArray *array, char **serialized) cJSON *cField = cJSON_CreateObject(); if (cField == NULL) goto _exit; + if (!cJSON_AddItemToObject(json, "configs", cField)) goto _exit; + // cjson only support int32_t or double // string are used to prohibit the loss of precision for (int i = 0; i < sz; i++) { @@ -2749,7 +2760,6 @@ int32_t globalConfigSerialize(int32_t version, SArray *array, char **serialized) } } } - if (!cJSON_AddItemToObject(json, "configs", cField)) goto _exit; char *pSerialized = cJSON_Print(json); _exit: if (terrno != TSDB_CODE_SUCCESS) { @@ -2769,6 +2779,9 @@ int32_t localConfigSerialize(SArray *array, char **serialized) { cJSON *cField = cJSON_CreateObject(); if (cField == NULL) goto _exit; + + if (!cJSON_AddItemToObject(json, "configs", cField)) goto _exit; + // cjson only support int32_t or double // string are used to prohibit the loss of precision for (int i = 0; i < sz; i++) { @@ -2802,7 +2815,6 @@ int32_t localConfigSerialize(SArray *array, char **serialized) { } } } - if (!cJSON_AddItemToObject(json, "configs", cField)) goto _exit; char *pSerialized = cJSON_Print(json); _exit: if (terrno != TSDB_CODE_SUCCESS) { diff --git a/tests/system-test/0-others/persisit_config.py b/tests/system-test/0-others/persisit_config.py new file mode 100644 index 0000000000..b7d437de3d --- /dev/null +++ b/tests/system-test/0-others/persisit_config.py @@ -0,0 +1,105 @@ +import sys +import taos +import os +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes + +class TDTestCase: + updatecfgDict = { + "supportVnodes":"1000", + "tmqMaxTopicNum":"30", + "maxShellConns":"1000", + "monitorFqdn":"localhost:9033", + "tmqRowSize":"1000", + "ttlChangeOnWrite":"1", + "compressor":"1", + "statusInterval":"4", + } + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + self.index = 1 + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def cli_get_param_value(self, config_name): + tdSql.query("show dnode 1 variables;") + for row in tdSql.queryResult: + if config_name == row[1]: + tdLog.debug("Found variable '{}'".format(row[0])) + return row[2] + + def cfg(self, option, value): + cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + if os.system(cmd) != 0 : + tdLog.exit(cmd) + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + + cfgPath = buildPath + "/../sim/dnode%d/cfg" % (self.index) + self.cfgPath = cfgPath + + dataPath = buildPath + "/../sim/dnode%d/data" % (self.index) + self.dataPath = dataPath + + logPath = buildPath + "/../sim/dnode%d/log" % (self.index) + self.logPath = logPath + + tdLog.info("start to check cfg value load from cfg file {0}".format(cfgPath)) + + for name,expValue in self.updatecfgDict.items(): + actValue = self.cli_get_param_value(name) + tdLog.debug("Get {} value: {} Expect value: {}".format(name, actValue,expValue)) + assert str(actValue) == str(expValue) + + tdLog.info("rm -rf {0}".format(cfgPath)) + os.system("rm -rf {0}/*".format(cfgPath)) + + tdLog.info("rebuilt cfg file {0}".format(cfgPath)) + cfgName = cfgPath+"/taos.cfg" + os.system("touch {0}".format(cfgName)) + os.system("echo 'fqdn localhost' >> {0}".format(cfgName)) + os.system("echo 'dataDir {0}' >> {1}".format(dataPath, cfgName)) + os.system("echo 'logDir {0}' >> {1}".format(logPath, cfgName)) + + tdDnodes.stop(1) + tdLog.info("restart taosd") + tdDnodes.start(1) + + tdLog.info("start to check cfg value load from mnd sdb") + + for name,expValue in self.updatecfgDict.items(): + actValue = self.cli_get_param_value(name) + tdLog.debug("Get {} value: {} Expect value: {}".format(name, actValue,expValue)) + assert str(actValue) == str(expValue) + + def stop(self): + tdSql.close() + # tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) + From 1aa4258aef47491e96a1b30b3c02a5ca65f322c4 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 2 Dec 2024 14:11:43 +0800 Subject: [PATCH 55/99] Fix mem error in readCfg. --- source/common/src/tglobal.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 3741c75db1..58fbbaf40f 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -2002,6 +2002,7 @@ int32_t readCfgFile(const char *path, bool isGlobal) { code = terrno; goto _exit; } + buf[fileSize] = 0; code = cfgDeserialize(array, buf, isGlobal); if (code != TSDB_CODE_SUCCESS) { uError("failed to deserialize config from %s since %s", filename, tstrerror(code)); From 475f54c75724dfe88493aef64b550afae392851d Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 4 Dec 2024 10:35:00 +0800 Subject: [PATCH 56/99] Fix ci mem error and add case. --- source/common/src/tglobal.c | 20 +- tests/parallel_test/cases.task | 2 + .../system-test/0-others/test_alter_config.py | 547 ++++++++++++++++++ 3 files changed, 556 insertions(+), 13 deletions(-) create mode 100644 tests/system-test/0-others/test_alter_config.py diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 58fbbaf40f..2e502474d5 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1934,9 +1934,10 @@ int32_t cfgDeserialize(SArray *array, char *buf, bool isGlobal) { if (pItem == NULL) { continue; } - if (strstr(item->name, "supportVnodes")) { - uDebug("supportVnodes:%d", pItem->valueint); + if (strstr(item->name, "syncLogBufferMemoryAllowed")) { + uDebug("syncLogBufferMemoryAllowed:%f", pItem->valuedouble); } + item->stype = CFG_STYPE_CFG_FILE; switch (item->dtype) { { case CFG_DTYPE_NONE: @@ -1952,13 +1953,14 @@ int32_t cfgDeserialize(SArray *array, char *buf, bool isGlobal) { break; case CFG_DTYPE_FLOAT: case CFG_DTYPE_DOUBLE: - item->fval = pItem->valuedouble; + item->fval = atoll(cJSON_GetStringValue(pItem)); break; case CFG_DTYPE_STRING: case CFG_DTYPE_DIR: case CFG_DTYPE_LOCALE: case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: + taosMemoryFree(item->str); item->str = taosStrdup(pItem->valuestring); break; } @@ -2076,7 +2078,6 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile if (!tsc) { TAOS_CHECK_GOTO(taosSetTfsCfg(tsCfg), &lino, _exit); - TAOS_CHECK_GOTO(taosUpdateServerCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(tryLoadCfgFromDataDir(tsCfg), &lino, _exit); } @@ -2084,6 +2085,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit); } else { TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit); + TAOS_CHECK_GOTO(taosUpdateServerCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosSetServerCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosSetReleaseCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosSetTfsCfg(tsCfg), &lino, _exit); @@ -2255,7 +2257,6 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"monitorForceV2", &tsMonitorForceV2}, {"monitorLogProtocol", &tsMonitorLogProtocol}, {"monitorMaxLogs", &tsMonitorMaxLogs}, - {"audit", &tsEnableAudit}, {"auditCreateTable", &tsEnableAuditCreateTable}, {"auditInterval", &tsAuditInterval}, {"slowLogThreshold", &tsSlowLogThreshold}, @@ -2268,7 +2269,6 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"maxRange", &tsMaxRange}, {"maxTsmaNum", &tsMaxTsmaNum}, {"queryRsmaTolerance", &tsQueryRsmaTolerance}, - {"countAlwaysReturnValue", &tsCountAlwaysReturnValue}, {"uptimeInterval", &tsUptimeInterval}, {"slowLogMaxLen", &tsSlowLogMaxLen}, @@ -2293,14 +2293,12 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"enableCoreFile", &tsEnableCoreFile}, {"telemetryInterval", &tsTelemInterval}, - {"telemetryPort", &tsTelemPort}, {"cacheLazyLoadThreshold", &tsCacheLazyLoadThreshold}, {"checkpointInterval", &tsStreamCheckpointInterval}, {"concurrentCheckpoint", &tsMaxConcurrentCheckpoint}, {"retentionSpeedLimitMB", &tsRetentionSpeedLimitMB}, - {"trimVDbIntervalSec", &tsTrimVDbIntervalSec}, {"ttlChangeOnWrite", &tsTtlChangeOnWrite}, {"logKeepDays", &tsLogKeepDays}, @@ -2330,7 +2328,6 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"s3BlockCacheSize", &tsS3BlockCacheSize}, {"s3PageCacheSize", &tsS3PageCacheSize}, {"s3UploadDelaySec", &tsS3UploadDelaySec}, - {"mndSdbWriteDelta", &tsMndSdbWriteDelta}, {"mndLogRetention", &tsMndLogRetention}, {"supportVnodes", &tsNumOfSupportVnodes}, {"experimental", &tsExperimental}, @@ -2342,7 +2339,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"rpcQueueMemoryAllowed", &tsQueueMemoryAllowed}, {"shellActivityTimer", &tsShellActivityTimer}, {"timeToGetAvailableConn", &tsTimeToGetAvailableConn}, - {"tsReadTimeout", &tsReadTimeout}, + {"readTimeout", &tsReadTimeout}, {"safetyCheckLevel", &tsSafetyCheckLevel}, {"bypassFlag", &tsBypassFlag}}; @@ -2589,9 +2586,6 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { {"queryPlannerTrace", &tsQueryPlannerTrace}, {"queryNodeChunkSize", &tsQueryNodeChunkSize}, {"queryUseNodeAllocator", &tsQueryUseNodeAllocator}, - {"randErrorChance", &tsRandErrChance}, - {"randErrorDivisor", &tsRandErrDivisor}, - {"randErrorScope", &tsRandErrScope}, {"smlDot2Underline", &tsSmlDot2Underline}, {"shellActivityTimer", &tsShellActivityTimer}, {"useAdapter", &tsUseAdapter}, diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 0cc84ad0ec..1d77057dc4 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -383,6 +383,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/test_hot_refresh_configurations.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/subscribe_stream_privilege.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/empty_identifier.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/persisit_config.py + ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/composite_primary_key_create.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/composite_primary_key_insert.py diff --git a/tests/system-test/0-others/test_alter_config.py b/tests/system-test/0-others/test_alter_config.py new file mode 100644 index 0000000000..3cbfc88fc7 --- /dev/null +++ b/tests/system-test/0-others/test_alter_config.py @@ -0,0 +1,547 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import subprocess +import random +import time +import os +import platform +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * + + +class TDTestCase: + """This test case is used to veirfy hot refresh configurations + """ + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + self.configration_dic = { + "cli": [ + ], + "svr": [ + { + "name": "audit", + "value": 0, + "category": "global" + }, + { + "name": "asyncLog", + "value": 0, + "category": "local" + }, + { + "name": "disableStream", + "value": 1, + "category": "global" + }, + { + "name": "enableWhiteList", + "value": 1, + "category": "global" + }, + { + "name": "statusInterval", + "value": 3, + "category": "global" + }, + { + "name": "telemetryReporting", + "value" : 1, + "category" : "global" + }, + { + "name" : "monitor", + "value" : 0, + "category" : "global" + }, + { + "name" : "monitorInterval", + "value" : 3, + "category" : "global" + }, + { + "name" : "monitorComp", + "value" : 1, + "category" : "global" + }, + { + "name" : "monitorForceV2", + "value" : 0, + "category" : "global" + }, + { + "name" : "monitorLogProtocol", + "value" : 1, + "category" : "global" + }, + { + "name": "monitorMaxLogs", + "value": 1000, + "category": "global" + }, + { + "name": "auditCreateTable", + "value": 0, + "category": "global" + }, + { + "name": "auditInterval", + "value": 4000, + "category": "global" + }, + { + "name": "slowLogThreshold", + "value": 20, + "category": "global" + }, + { + "name": "compressMsgSize", + "value": 0, + "category": "global" + }, + { + "name": "compressor", + "value": "GZIP_COMPRESSOR", + "category": "global" + }, + { + "name": "curRange", + "value": 200, + "category": "global" + }, + { + "name": "fPrecision", + "value": "1000.000000", + "category": "global" + }, + { + "name": "dPrecision", + "value": "1000.000000", + "category": "global" + }, + { + "name": "ifAdtFse", + "value": 1, + "category": "global" + }, + { + "name": "maxRange", + "value": 1000, + "category": "global" + }, + { + "name": "maxTsmaNum", + "value": 2, + "category": "global" + }, + { + "name": "queryRsmaTolerance", + "value": 2000, + "category": "global" + }, + { + "name": "uptimeInterval", + "value": 600, + "category": "global" + }, + { + "name": "slowLogMaxLen", + "value": 8192, + "category": "global" + }, + { + "name": "slowLogScope", + "value": "insert", + "category": "global" + }, + { + "name": "slowLogExceptDb", + "value": "db1", + "category": "global" + }, + { + "name": "mndSdbWriteDelta", + "value": 1000, + "category": "global" + }, + { + "name": "minDiskFreeSize", + "value": 100*1024*1024, + "category": "global" + }, + { + "name": "randErrorChance", + "value": 5, + "category": "global" + }, + { + "name": "randErrorDivisor", + "value": 20001, + "category": "global" + }, + { + "name": "randErrorScope", + "value": 8, + "category": "global" + }, + { + "name": "syncLogBufferMemoryAllowed", + "value": 1024 * 1024 * 20 * 10, + "category": "global" + }, + { + "name": "resolveFQDNRetryTime", + "value": 500, + "category": "global" + }, + { + "name": "syncElectInterval", + "value": 50000, + "category": "global" + }, + { + "name": "syncHeartbeatInterval", + "value": 3000, + "category": "global" + }, + { + "name": "syncHeartbeatTimeout", + "value": 40000, + "category": "global" + }, + { + "name": "syncSnapReplMaxWaitN", + "value": 32, + "category": "global" + }, + { + "name": "walFsyncDataSizeLimit", + "value": 200 * 1024 * 1024, + "category": "global" + }, + { + "name": "numOfCores", + "value": "30.000000", + "category": "global" + }, + { + "name": "enableCoreFile", + "value": 0, + "category": "global" + }, + { + "name": "telemetryInterval", + "value": 6000, + "category": "global" + }, + { + "name": "cacheLazyLoadThreshold", + "value": 1000, + "category": "global" + }, + { + "name": "checkpointInterval", + "value": 120, + "category": "global" + }, + { + "name": "concurrentCheckpoint", + "value": 3, + "category": "global" + }, + { + "name": "retentionSpeedLimitMB", + "value": 24, + "category": "global" + }, + { + "name": "ttlChangeOnWrite", + "value": 1, + "category": "global" + }, + { + "name": "logKeepDays", + "value": 30, + "category": "local" + }, + { + "name": "maxStreamBackendCache", + "value": 256, + "category": "global" + }, + { + "name": "mqRebalanceInterval", + "value": 30, + "category": "global" + }, + { + "name": "numOfLogLines", + "value": 20000000, + "category": "local" + }, + { + "name": "queryRspPolicy", + "value": 1, + "category": "global" + }, + { + "name": "timeseriesThreshold", + "value": 100, + "category": "global" + }, + { + "name": "tmqMaxTopicNum", + "value": 30, + "category": "global" + }, + { + "name": "tmqRowSize", + "value": 8192, + "category": "global" + }, + { + "name": "transPullupInterval", + "value": 4, + "category": "global" + }, + { + "name": "compactPullupInterval", + "value": 20, + "category": "global" + }, + { + "name": "trimVDbIntervalSec", + "value": 7200, + "category": "global" + }, + { + "name": "ttlBatchDropNum", + "value": 20000, + "category": "global" + }, + { + "name": "ttlFlushThreshold", + "value": 200, + "category": "global" + }, + { + "name": "ttlPushInterval", + "value": 20, + "category": "global" + }, + { + "name": "ttlUnit", + "value": 86500, + "category": "global" + }, + { + "name": "udf", + "value": 0, + "category": "global" + }, + # { + # "name": "udfdLdLibPath", + # "value": 1000, + # "category": "global" + # }, + # { + # "name": "udfdResFuncs", + # "value": 1000, + # "category": "global" + # }, + # { + # "name": "s3Accesskey", + # "value": 1000, + # "category": "global" + # }, + # { + # "name": "s3BucketName", + # "value": 1000, + # "category": "global" + # }, + # { + # "name": "s3Endpoint", + # "value": 1000, + # "category": "global" + # }, + { + "name": "s3MigrateIntervalSec", + "value": 1800, + "category": "global" + }, + { + "name": "s3MigrateEnabled", + "value": 1, + "category": "global" + }, + # { + # "name": "s3BlockCacheSize", + # "value": 32, + # "category": "global" + # }, + { + "name": "s3PageCacheSize", + "value": 8192, + "category": "global" + }, + { + "name": "s3UploadDelaySec", + "value": 30, + "category": "global" + }, + { + "name": "mndLogRetention", + "value": 1000, + "category": "global" + }, + { + "name": "supportVnodes", + "value": 128, + "category": "global" + }, + { + "name": "experimental", + "value": 0, + "category": "global" + }, + { + "name": "maxTsmaNum", + "value": 2, + "category": "global" + }, + { + "name": "maxShellConns", + "value": 25000, + "category": "global" + }, + { + "name": "numOfRpcSessions", + "value": 15000, + "category": "global" + }, + { + "name": "numOfRpcThreads", + "value": 2, + "category": "global" + }, + { + "name": "rpcQueueMemoryAllowed", + "value": 1024*1024*20*10, + "category": "global" + }, + { + "name": "shellActivityTimer", + "value": 2, + "category": "global" + }, + { + "name": "timeToGetAvailableConn", + "value": 200000, + "category": "global" + }, + { + "name": "readTimeout", + "value": 800, + "category": "global" + }, + { + "name": "safetyCheckLevel", + "value": 2, + "category": "global" + }, + { + "name": "bypassFlag", + "value": 4, + "category": "global" + } + ] + } + def cli_get_param_value(self, config_name): + tdSql.query("show local variables;") + for row in tdSql.queryResult: + if config_name == row[0]: + return row[1] + + def svr_get_param_value(self, config_name): + tdSql.query("show dnode 1 variables;") + for row in tdSql.queryResult: + if config_name == row[1]: + return row[2] + + def configuration_alter(self): + for key in self.configration_dic: + if "svr" == key: + for item in self.configration_dic[key]: + name = item["name"] + value = item["value"] + category = item["category"] + if category == "global": + tdSql.execute(f'alter all dnodes "{name} {value}";') + else: + tdSql.execute(f'alter dnode 1 "{name} {value}";') + elif "cli" == key: + for item in self.configration_dic[key]: + name = item["name"] + value = item["value"] + category = item["category"] + if category == "global": + tdSql.execute(f'alter all dnodes "{name} {value}";') + else: + tdSql.execute(f'alter local "{name} {value}";') + else: + raise Exception(f"unknown key: {key}") + + def run(self): + self.configuration_alter() + for key in self.configration_dic: + if "cli" == key: + for item in self.configration_dic[key]: + actVal = self.cli_get_param_value(item["name"]) + assert str(actVal) == str(item["value"]), f"tem name: {item['name']}, Expected value: {item['value']}, actual value: {actVal}" + elif "svr" == key: + for item in self.configration_dic[key]: + actVal = self.svr_get_param_value(item["name"]) + assert str(actVal) == str(item["value"]), f"tem name: {item['name']}, Expected value: {item['value']}, actual value: {actVal}" + else: + raise Exception(f"unknown key: {key}") + + tdLog.info("stop and restart taosd") + tdDnodes.stop(1) + tdDnodes.start(1) + + for key in self.configration_dic: + if "cli" == key: + for item in self.configration_dic[key]: + actVal = self.cli_get_param_value(item["name"]) + assert str(actVal) == str(item["oldVal"]), f"item name: {item['name']}, Expected value: {item['value']}, actual value: {actVal}" + elif "svr" == key: + for item in self.configration_dic[key]: + actVal = self.svr_get_param_value(item["name"]) + assert str(actVal) == str(item["value"]), f"item name: {item['name']}, Expected value: {item['value']}, actual value: {actVal}" + else: + raise Exception(f"unknown key: {key}") + + def stop(self): + tdSql.close() + # tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From eb202e556a9595e66a26a73509ce1b7f5b7b7fe2 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 4 Dec 2024 19:32:21 +0800 Subject: [PATCH 57/99] Fix sdb lost val while kill taosd. --- include/util/tconfig.h | 3 +- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 1 + source/dnode/mnode/impl/inc/mndDef.h | 3 +- source/dnode/mnode/impl/src/mndConfig.c | 98 +-- source/dnode/mnode/impl/src/mndDef.c | 22 +- source/util/src/tconfig.c | 45 ++ tests/army/alter/test_alter_config.py | 556 ++++++++++++++++++ .../system-test/0-others/test_alter_config.py | 3 +- 8 files changed, 675 insertions(+), 56 deletions(-) create mode 100644 tests/army/alter/test_alter_config.py diff --git a/include/util/tconfig.h b/include/util/tconfig.h index a8e3343ad0..8126e7e6a9 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -114,7 +114,8 @@ typedef struct SConfigIter SConfigIter; int32_t cfgInit(SConfig **ppCfg); int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const void *sourceStr); -int32_t cfgLoadFromArray(SConfig *pCfg, SArray *pArgs); // SConfigPair +int32_t cfgLoadFromArray(SConfig *pCfg, SArray *pArgs); // SConfigPair +int32_t cfgUpdateFromArray(SConfig *pCfg, SArray *pArgs); // SConfigItem void cfgCleanup(SConfig *pCfg); int32_t cfgGetSize(SConfig *pCfg); SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 822a89810d..c17d3affb3 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -330,6 +330,7 @@ static void dmProcessConfigRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { } } if (needUpdate) { + code = cfgUpdateFromArray(tsCfg, configRsp.array); code = setAllConfigs(tsCfg); if (code != TSDB_CODE_SUCCESS) { dError("failed to set all configs since %s", tstrerror(code)); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 8bd81f612f..486b18ac29 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -319,13 +319,12 @@ typedef struct { typedef struct { char name[CFG_NAME_MAX_LEN]; ECfgDataType dtype; - int32_t strLen; union { bool bval; float fval; int32_t i32; int64_t i64; - char* str; + char str[TSDB_CONFIG_VALUE_LEN]; }; } SConfigObj; diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index b9108776b3..5b21131df2 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -32,7 +32,7 @@ enum CfgAlterType { static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pInMCfgReq, int32_t optLen, int32_t *pOutValue); static int32_t cfgUpdateItem(SConfigItem *pItem, SConfigObj *obj); -static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue); +static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue, ECfgDataType dtype); static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq); static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessConfigReq(SRpcMsg *pReq); @@ -66,7 +66,7 @@ SSdbRaw *mnCfgActionEncode(SConfigObj *obj) { terrno = TSDB_CODE_OUT_OF_MEMORY; char buf[30]; - int32_t sz = sizeof(SConfigObj) + obj->strLen + CFG_RESERVE_SIZE; + int32_t sz = sizeof(SConfigObj) + CFG_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_CFG, CFG_VER_NUMBER, sz); if (pRaw == NULL) goto _OVER; @@ -75,7 +75,6 @@ SSdbRaw *mnCfgActionEncode(SConfigObj *obj) { strncpy(name, obj->name, CFG_NAME_MAX_LEN); SDB_SET_BINARY(pRaw, dataPos, name, CFG_NAME_MAX_LEN, _OVER) SDB_SET_INT32(pRaw, dataPos, obj->dtype, _OVER) - SDB_SET_INT32(pRaw, dataPos, obj->strLen, _OVER) switch (obj->dtype) { case CFG_DTYPE_NONE: break; @@ -97,9 +96,7 @@ SSdbRaw *mnCfgActionEncode(SConfigObj *obj) { case CFG_DTYPE_LOCALE: case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: - if (obj->str != NULL) { - SDB_SET_BINARY(pRaw, dataPos, obj->str, obj->strLen, _OVER) - } + SDB_SET_BINARY(pRaw, dataPos, obj->str, TSDB_CONFIG_VALUE_LEN, _OVER) break; } SDB_SET_RESERVE(pRaw, dataPos, CFG_RESERVE_SIZE, _OVER) @@ -141,7 +138,6 @@ SSdbRow *mndCfgActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, obj->name, CFG_NAME_MAX_LEN, _OVER) SDB_GET_INT32(pRaw, dataPos, (int32_t *)&obj->dtype, _OVER) - SDB_GET_INT32(pRaw, dataPos, &obj->strLen, _OVER) switch (obj->dtype) { case CFG_DTYPE_NONE: break; @@ -163,10 +159,7 @@ SSdbRow *mndCfgActionDecode(SSdbRaw *pRaw) { case CFG_DTYPE_LOCALE: case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: - if (len > 0) { - obj->str = taosMemoryMalloc(len); - SDB_GET_BINARY(pRaw, dataPos, obj->str, len, _OVER) - } + SDB_GET_BINARY(pRaw, dataPos, obj->str, TSDB_CONFIG_VALUE_LEN, _OVER) break; } terrno = TSDB_CODE_SUCCESS; @@ -175,7 +168,6 @@ _OVER: if (terrno != 0) { mError("cfg failed to decode from raw:%p since %s", pRaw, terrstr()); taosMemoryFreeClear(pRow); - taosMemoryFreeClear(obj->str); return NULL; } @@ -195,7 +187,31 @@ static int32_t mndCfgActionDelete(SSdb *pSdb, SConfigObj *obj) { static int32_t mndCfgActionUpdate(SSdb *pSdb, SConfigObj *pOld, SConfigObj *pNew) { mTrace("cfg:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); - return 0; + switch (pNew->dtype) { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + pOld->bval = pNew->bval; + break; + case CFG_DTYPE_INT32: + pOld->i32 = pNew->i32; + break; + case CFG_DTYPE_INT64: + pOld->i64 = pNew->i64; + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + pOld->fval = pNew->fval; + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + tstrncpy(pOld->str, pNew->str, TSDB_CONFIG_VALUE_LEN); + break; + } + return TSDB_CODE_SUCCESS; } static int32_t mndCfgActionDeploy(SMnode *pMnode) { return mndInitWriteCfg(pMnode); } @@ -272,7 +288,6 @@ int32_t mndInitWriteCfg(SMnode *pMnode) { taosMemoryFree(versionObj); goto _OVER; } - taosMemoryFree(versionObj->str); taosMemoryFree(versionObj); sz = taosArrayGetSize(taosGetGlobalCfg(tsCfg)); @@ -282,9 +297,6 @@ int32_t mndInitWriteCfg(SMnode *pMnode) { if ((code = mndSetCreateConfigCommitLogs(pTrans, obj)) != 0) { mError("failed to init mnd config:%s, since %s", item->name, terrstr()); } - if (obj->strLen > 0) { - taosMemoryFree(obj->str); - } taosMemoryFree(obj); } if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER; @@ -297,7 +309,7 @@ _OVER: int32_t mndInitReadCfg(SMnode *pMnode) { int32_t code = 0; int32_t sz = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "init-read-config"); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "init-read-config"); if (pTrans == NULL) { mError("failed to init read cfg in create trans, since %s", terrstr()); goto _OVER; @@ -374,7 +386,8 @@ int32_t cfgUpdateItem(SConfigItem *pItem, SConfigObj *obj) { case CFG_DTYPE_STRING: { if (obj->str != NULL) { taosMemoryFree(pItem->str); - pItem->str = obj->str; + pItem->str = taosMemoryMalloc(strlen(obj->str) + 1); + tstrncpy(pItem->str, obj->str, strlen(obj->str) + 1); } break; } @@ -510,7 +523,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { goto _err_out; } if (pItem->category == CFG_CATEGORY_GLOBAL) { - TAOS_CHECK_GOTO(mndConfigUpdateTrans(pMnode, dcfgReq.config, dcfgReq.value), &lino, _err_out); + TAOS_CHECK_GOTO(mndConfigUpdateTrans(pMnode, dcfgReq.config, dcfgReq.value, pItem->dtype), &lino, _err_out); } _send_req : @@ -565,31 +578,38 @@ _err: TAOS_RETURN(code); } -static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue) { - int32_t code = -1; - int32_t lino = -1; - SConfigObj *pVersion = sdbAcquire(pMnode->pSdb, SDB_CFG, "tsmmConfigVersion"); - if (pVersion == NULL) { - mError("failed to acquire tsmmConfigVersion while update config, since %s", terrstr()); - code = terrno; - goto _OVER; - } - pVersion->i32 = ++tsmmConfigVersion; - SConfigObj *pObj = sdbAcquire(pMnode->pSdb, SDB_CFG, name); - if (pObj == NULL) { - mError("failed to acquire mnd config:%s while update config, since %s", name, terrstr()); - code = terrno; - goto _OVER; - } - TAOS_CHECK_GOTO(mndUpdateObj(pObj, name, pValue), &lino, _OVER); +static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue, ECfgDataType dtype) { + int32_t code = -1; + int32_t lino = -1; + // SConfigObj *pVersion = sdbAcquire(pMnode->pSdb, SDB_CFG, "tsmmConfigVersion"); + // if (pVersion == NULL) { + // mError("failed to acquire tsmmConfigVersion while update config, since %s", terrstr()); + // code = terrno; + // goto _OVER; + // } + // pVersion->i32 = ++tsmmConfigVersion; + // SConfigObj *pObj = sdbAcquire(pMnode->pSdb, SDB_CFG, name); + // if (pObj == NULL) { + // mError("failed to acquire mnd config:%s while update config, since %s", name, terrstr()); + // code = terrno; + // goto _OVER; + // } + SConfigObj pVersion = {0}, pObj = {0}; + pVersion.i32 = ++tsmmConfigVersion; + strncpy(pVersion.name, "tsmmConfigVersion", CFG_NAME_MAX_LEN); + + pObj.dtype = dtype; + strncpy(pObj.name, name, CFG_NAME_MAX_LEN); + + TAOS_CHECK_GOTO(mndUpdateObj(&pObj, name, pValue), &lino, _OVER); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_ARBGROUP, NULL, "update-config"); if (pTrans == NULL) { if (terrno != 0) code = terrno; goto _OVER; } mInfo("trans:%d, used to update config:%s to value:%s", pTrans->id, name, pValue); - TAOS_CHECK_GOTO(mndSetCreateConfigCommitLogs(pTrans, pVersion), &lino, _OVER); - TAOS_CHECK_GOTO(mndSetCreateConfigCommitLogs(pTrans, pObj), &lino, _OVER); + TAOS_CHECK_GOTO(mndSetCreateConfigCommitLogs(pTrans, &pVersion), &lino, _OVER); + TAOS_CHECK_GOTO(mndSetCreateConfigCommitLogs(pTrans, &pObj), &lino, _OVER); if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER; code = 0; _OVER: diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index c26e5ac171..f257991381 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -758,17 +758,15 @@ SConfigObj *mndInitConfigObj(SConfigItem *pItem) { case CFG_DTYPE_LOCALE: case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: - pObj->str = taosStrdup(pItem->str); - pObj->strLen = strlen(pItem->str) + 1; + tstrncpy(pObj->str, pItem->str, TSDB_CONFIG_VALUE_LEN); break; } return pObj; } -int32_t mndUpdateObj(SConfigObj *pObj, const char *name, char *value) { +int32_t mndUpdateObj(SConfigObj *pObjNew, const char *name, char *value) { int32_t code = 0; - - switch (pObj->dtype) { + switch (pObjNew->dtype) { case CFG_DTYPE_BOOL: { bool tmp = false; if (strcasecmp(value, "true") == 0) { @@ -777,26 +775,26 @@ int32_t mndUpdateObj(SConfigObj *pObj, const char *name, char *value) { if (atoi(value) > 0) { tmp = true; } - pObj->bval = tmp; + pObjNew->bval = tmp; break; } case CFG_DTYPE_INT32: { int32_t ival; TAOS_CHECK_RETURN(taosStrHumanToInt32(value, &ival)); - pObj->i32 = ival; + pObjNew->i32 = ival; break; } case CFG_DTYPE_INT64: { int64_t ival; TAOS_CHECK_RETURN(taosStrHumanToInt64(value, &ival)); - pObj->i64 = ival; + pObjNew->i64 = ival; break; } case CFG_DTYPE_FLOAT: case CFG_DTYPE_DOUBLE: { float dval = 0; TAOS_CHECK_RETURN(parseCfgReal(value, &dval)); - pObj->fval = dval; + pObjNew->fval = dval; break; } case CFG_DTYPE_DIR: @@ -804,10 +802,8 @@ int32_t mndUpdateObj(SConfigObj *pObj, const char *name, char *value) { case CFG_DTYPE_CHARSET: case CFG_DTYPE_LOCALE: case CFG_DTYPE_STRING: { - char *tmp = taosStrdup(value); - taosMemoryFreeClear(pObj->str); - pObj->str = tmp; - pObj->strLen = strlen(value) + 1; + strncpy(pObjNew->str, value, strlen(value)); + pObjNew->str[strlen(value)] = 0; break; } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index a84e77b9bc..029ada0a75 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -95,6 +95,51 @@ int32_t cfgLoadFromArray(SConfig *pCfg, SArray *pArgs) { return TSDB_CODE_SUCCESS; } +int32_t cfgUpdateFromArray(SConfig *pCfg, SArray *pArgs) { + int32_t size = taosArrayGetSize(pArgs); + for (int32_t i = 0; i < size; ++i) { + SConfigItem *pItemNew = taosArrayGet(pArgs, i); + + (void)taosThreadMutexLock(&pCfg->lock); + + SConfigItem *pItemOld = cfgGetItem(pCfg, pItemNew->name); + if (pItemOld == NULL) { + (void)taosThreadMutexUnlock(&pCfg->lock); + TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); + } + char value[30]; + switch (pItemNew->dtype) { + case CFG_DTYPE_BOOL: + strncpy(value, pItemNew->bval ? "true" : "false", sizeof(value) - 1); + break; + case CFG_DTYPE_INT32: + sprintf(value, "%d", pItemNew->i32); + break; + case CFG_DTYPE_INT64: + sprintf(value, "%" PRId64, pItemNew->i64); + break; + case CFG_DTYPE_FLOAT: + sprintf(value, "%f", pItemNew->fval); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + strcpy(value, pItemNew->str); + default: + break; + } + value[sizeof(value) - 1] = '\0'; + + TAOS_CHECK_RETURN(cfgSetItemVal(pItemOld, pItemNew->name, value, CFG_STYPE_ARG_LIST)); + + (void)taosThreadMutexUnlock(&pCfg->lock); + } + + return TSDB_CODE_SUCCESS; +} + void cfgItemFreeVal(SConfigItem *pItem) { if (pItem->dtype == CFG_DTYPE_STRING || pItem->dtype == CFG_DTYPE_DIR || pItem->dtype == CFG_DTYPE_LOCALE || pItem->dtype == CFG_DTYPE_CHARSET || pItem->dtype == CFG_DTYPE_TIMEZONE) { diff --git a/tests/army/alter/test_alter_config.py b/tests/army/alter/test_alter_config.py new file mode 100644 index 0000000000..7395268549 --- /dev/null +++ b/tests/army/alter/test_alter_config.py @@ -0,0 +1,556 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time + +import taos +import frame +import frame.etool + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame.epath import * +from frame.srvCtl import * +from frame import * + +class TDTestCase: + """This test case is used to veirfy hot refresh configurations + """ + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + self.configration_dic = { + "cli": [ + ], + "svr": [ + { + "name": "audit", + "value": 0, + "category": "global" + }, + { + "name": "asyncLog", + "value": 0, + "category": "local" + }, + { + "name": "disableStream", + "value": 1, + "category": "global" + }, + { + "name": "enableWhiteList", + "value": 1, + "category": "global" + }, + { + "name": "statusInterval", + "value": 3, + "category": "global" + }, + { + "name": "telemetryReporting", + "value" : 1, + "category" : "global" + }, + { + "name" : "monitor", + "value" : 0, + "category" : "global" + }, + { + "name" : "monitorInterval", + "value" : 3, + "category" : "global" + }, + { + "name" : "monitorComp", + "value" : 1, + "category" : "global" + }, + { + "name" : "monitorForceV2", + "value" : 0, + "category" : "global" + }, + { + "name" : "monitorLogProtocol", + "value" : 1, + "category" : "global" + }, + { + "name": "monitorMaxLogs", + "value": 1000, + "category": "global" + }, + { + "name": "auditCreateTable", + "value": 0, + "category": "global" + }, + { + "name": "auditInterval", + "value": 4000, + "category": "global" + }, + { + "name": "slowLogThreshold", + "value": 20, + "category": "global" + }, + { + "name": "compressMsgSize", + "value": 0, + "category": "global" + }, + { + "name": "compressor", + "value": "GZIP_COMPRESSOR", + "category": "global" + }, + { + "name": "curRange", + "value": 200, + "category": "global" + }, + { + "name": "fPrecision", + "value": "1000.000000", + "category": "global" + }, + { + "name": "dPrecision", + "value": "1000.000000", + "category": "global" + }, + { + "name": "ifAdtFse", + "value": 1, + "category": "global" + }, + { + "name": "maxRange", + "value": 1000, + "category": "global" + }, + { + "name": "maxTsmaNum", + "value": 2, + "category": "global" + }, + { + "name": "queryRsmaTolerance", + "value": 2000, + "category": "global" + }, + { + "name": "uptimeInterval", + "value": 600, + "category": "global" + }, + { + "name": "slowLogMaxLen", + "value": 8192, + "category": "global" + }, + { + "name": "slowLogScope", + "value": "insert", + "category": "global" + }, + { + "name": "slowLogExceptDb", + "value": "db1", + "category": "global" + }, + { + "name": "mndSdbWriteDelta", + "value": 1000, + "category": "global" + }, + { + "name": "minDiskFreeSize", + "value": 100*1024*1024, + "category": "global" + }, + { + "name": "randErrorChance", + "value": 5, + "category": "global" + }, + { + "name": "randErrorDivisor", + "value": 20001, + "category": "global" + }, + { + "name": "randErrorScope", + "value": 8, + "category": "global" + }, + { + "name": "syncLogBufferMemoryAllowed", + "value": 1024 * 1024 * 20 * 10, + "category": "global" + }, + { + "name": "resolveFQDNRetryTime", + "value": 500, + "category": "global" + }, + { + "name": "syncElectInterval", + "value": 50000, + "category": "global" + }, + { + "name": "syncHeartbeatInterval", + "value": 3000, + "category": "global" + }, + { + "name": "syncHeartbeatTimeout", + "value": 40000, + "category": "global" + }, + { + "name": "syncSnapReplMaxWaitN", + "value": 32, + "category": "global" + }, + { + "name": "walFsyncDataSizeLimit", + "value": 200 * 1024 * 1024, + "category": "global" + }, + { + "name": "numOfCores", + "value": "30.000000", + "category": "global" + }, + { + "name": "enableCoreFile", + "value": 0, + "category": "global" + }, + { + "name": "telemetryInterval", + "value": 6000, + "category": "global" + }, + { + "name": "cacheLazyLoadThreshold", + "value": 1000, + "category": "global" + }, + { + "name": "checkpointInterval", + "value": 120, + "category": "global" + }, + { + "name": "concurrentCheckpoint", + "value": 3, + "category": "global" + }, + { + "name": "retentionSpeedLimitMB", + "value": 24, + "category": "global" + }, + { + "name": "ttlChangeOnWrite", + "value": 1, + "category": "global" + }, + { + "name": "logKeepDays", + "value": 30, + "category": "local" + }, + { + "name": "maxStreamBackendCache", + "value": 256, + "category": "global" + }, + { + "name": "mqRebalanceInterval", + "value": 30, + "category": "global" + }, + { + "name": "numOfLogLines", + "value": 20000000, + "category": "local" + }, + { + "name": "queryRspPolicy", + "value": 1, + "category": "global" + }, + { + "name": "timeseriesThreshold", + "value": 100, + "category": "global" + }, + { + "name": "tmqMaxTopicNum", + "value": 30, + "category": "global" + }, + { + "name": "tmqRowSize", + "value": 8192, + "category": "global" + }, + { + "name": "transPullupInterval", + "value": 4, + "category": "global" + }, + { + "name": "compactPullupInterval", + "value": 20, + "category": "global" + }, + { + "name": "trimVDbIntervalSec", + "value": 7200, + "category": "global" + }, + { + "name": "ttlBatchDropNum", + "value": 20000, + "category": "global" + }, + { + "name": "ttlFlushThreshold", + "value": 200, + "category": "global" + }, + { + "name": "ttlPushInterval", + "value": 20, + "category": "global" + }, + { + "name": "ttlUnit", + "value": 86500, + "category": "global" + }, + { + "name": "udf", + "value": 0, + "category": "global" + }, + # { + # "name": "udfdLdLibPath", + # "value": 1000, + # "category": "global" + # }, + # { + # "name": "udfdResFuncs", + # "value": 1000, + # "category": "global" + # }, + # { + # "name": "s3Accesskey", + # "value": 1000, + # "category": "global" + # }, + # { + # "name": "s3BucketName", + # "value": 1000, + # "category": "global" + # }, + # { + # "name": "s3Endpoint", + # "value": 1000, + # "category": "global" + # }, + { + "name": "s3MigrateIntervalSec", + "value": 1800, + "category": "global" + }, + { + "name": "s3MigrateEnabled", + "value": 1, + "category": "global" + }, + # { + # "name": "s3BlockCacheSize", + # "value": 32, + # "category": "global" + # }, + { + "name": "s3PageCacheSize", + "value": 8192, + "category": "global" + }, + { + "name": "s3UploadDelaySec", + "value": 30, + "category": "global" + }, + { + "name": "mndLogRetention", + "value": 1000, + "category": "global" + }, + { + "name": "supportVnodes", + "value": 128, + "category": "global" + }, + { + "name": "experimental", + "value": 0, + "category": "global" + }, + { + "name": "maxTsmaNum", + "value": 2, + "category": "global" + }, + { + "name": "maxShellConns", + "value": 25000, + "category": "global" + }, + { + "name": "numOfRpcSessions", + "value": 15000, + "category": "global" + }, + { + "name": "numOfRpcThreads", + "value": 2, + "category": "global" + }, + { + "name": "rpcQueueMemoryAllowed", + "value": 1024*1024*20*10, + "category": "global" + }, + { + "name": "shellActivityTimer", + "value": 2, + "category": "global" + }, + { + "name": "timeToGetAvailableConn", + "value": 200000, + "category": "global" + }, + { + "name": "readTimeout", + "value": 800, + "category": "global" + }, + { + "name": "safetyCheckLevel", + "value": 2, + "category": "global" + }, + { + "name": "bypassFlag", + "value": 4, + "category": "global" + } + ] + } + def cli_get_param_value(self, config_name): + res = tdSql.getResult("show local variables") + for row in res: + if config_name == row[0]: + return row[1] + + def svr_get_param_value(self, config_name): + res = tdSql.getResult("show dnode 1 variables") + for row in res: + if config_name == row[1]: + return row[2] + + def configuration_alter(self): + for key in self.configration_dic: + if "svr" == key: + for item in self.configration_dic[key]: + name = item["name"] + value = item["value"] + category = item["category"] + if category == "global": + tdSql.execute(f'alter all dnodes "{name} {value}";') + else: + tdSql.execute(f'alter dnode 1 "{name} {value}";') + elif "cli" == key: + for item in self.configration_dic[key]: + name = item["name"] + value = item["value"] + category = item["category"] + if category == "global": + tdSql.execute(f'alter all dnodes "{name} {value}";') + else: + tdSql.execute(f'alter local "{name} {value}";') + else: + raise Exception(f"unknown key: {key}") + + def run(self): + self.configuration_alter() + for key in self.configration_dic: + if "cli" == key: + for item in self.configration_dic[key]: + actVal = self.cli_get_param_value(item["name"]) + assert str(actVal) == str(item["value"]), f"tem name: {item['name']}, Expected value: {item['value']}, actual value: {actVal}" + elif "svr" == key: + for item in self.configration_dic[key]: + actVal = self.svr_get_param_value(item["name"]) + assert str(actVal) == str(item["value"]), f"tem name: {item['name']}, Expected value: {item['value']}, actual value: {actVal}" + else: + raise Exception(f"unknown key: {key}") + + tdLog.info("success to alter all configurations") + + tdLog.info("stop and restart taosd") + time.sleep(3) + sc.dnodeStopAll() + sc.dnodeStart(1) + sc.dnodeStart(2) + sc.dnodeStart(3) + time.sleep(10) + + for key in self.configration_dic: + if "cli" == key: + for item in self.configration_dic[key]: + actVal = self.cli_get_param_value(item["name"]) + assert str(actVal) == str(item["oldVal"]), f"item name: {item['name']}, Expected value: {item['value']}, actual value: {actVal}" + elif "svr" == key: + for item in self.configration_dic[key]: + actVal = self.svr_get_param_value(item["name"]) + assert str(actVal) == str(item["value"]), f"item name: {item['name']}, Expected value: {item['value']}, actual value: {actVal}" + else: + raise Exception(f"unknown key: {key}") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/test_alter_config.py b/tests/system-test/0-others/test_alter_config.py index 3cbfc88fc7..14e77bb489 100644 --- a/tests/system-test/0-others/test_alter_config.py +++ b/tests/system-test/0-others/test_alter_config.py @@ -21,7 +21,8 @@ from util.sql import * from util.cases import * from util.dnodes import * from util.common import * - +from frame.srvCtl import * +from frame import * class TDTestCase: """This test case is used to veirfy hot refresh configurations From cb50b1e52c825d54ff0fd57b5f1579187e0e4619 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 5 Dec 2024 09:16:10 +0800 Subject: [PATCH 58/99] Fix some mem errors and add dmstop while verify failed. --- source/common/src/tglobal.c | 5 +++++ source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 1 + source/dnode/mnode/impl/src/mndConfig.c | 3 ++- source/util/src/tconfig.c | 17 ++++++++--------- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 73cb2f7e27..a6bb249bc3 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -892,6 +892,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + SConfigItem *pItem = NULL; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "fPrecision"); + tsFPrecision = pItem->fval; TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "curRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); @@ -1998,6 +2001,8 @@ int32_t cfgDeserialize(SArray *array, char *buf, bool isGlobal) { } } } +_exit: + cJSON_Delete(pRoot); return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index c17d3affb3..a856a723c6 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -315,6 +315,7 @@ static void dmProcessConfigRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { } else { // log the difference configurations printConfigNotMatch(configRsp.array); + dmStop(); goto _exit; } } diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index 5b21131df2..f7065ddf09 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -387,7 +387,8 @@ int32_t cfgUpdateItem(SConfigItem *pItem, SConfigObj *obj) { if (obj->str != NULL) { taosMemoryFree(pItem->str); pItem->str = taosMemoryMalloc(strlen(obj->str) + 1); - tstrncpy(pItem->str, obj->str, strlen(obj->str) + 1); + tstrncpy(pItem->str, obj->str, strlen(obj->str)); + pItem->str[strlen(obj->str)] = 0; } break; } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 029ada0a75..7d82666c65 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -107,32 +107,31 @@ int32_t cfgUpdateFromArray(SConfig *pCfg, SArray *pArgs) { (void)taosThreadMutexUnlock(&pCfg->lock); TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); } - char value[30]; switch (pItemNew->dtype) { case CFG_DTYPE_BOOL: - strncpy(value, pItemNew->bval ? "true" : "false", sizeof(value) - 1); + pItemOld->bval = pItemNew->bval; break; case CFG_DTYPE_INT32: - sprintf(value, "%d", pItemNew->i32); + pItemOld->i32 = pItemNew->i32; break; case CFG_DTYPE_INT64: - sprintf(value, "%" PRId64, pItemNew->i64); + pItemOld->i64 = pItemNew->i64; break; case CFG_DTYPE_FLOAT: - sprintf(value, "%f", pItemNew->fval); + case CFG_DTYPE_DOUBLE: + pItemOld->fval = pItemNew->fval; break; case CFG_DTYPE_STRING: case CFG_DTYPE_DIR: case CFG_DTYPE_LOCALE: case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: - strcpy(value, pItemNew->str); + taosMemoryFree(pItemOld->str); + pItemOld->str = taosStrdup(pItemNew->str); + break; default: break; } - value[sizeof(value) - 1] = '\0'; - - TAOS_CHECK_RETURN(cfgSetItemVal(pItemOld, pItemNew->name, value, CFG_STYPE_ARG_LIST)); (void)taosThreadMutexUnlock(&pCfg->lock); } From cf5960d847190ddc6b8328c2b5b9cd09a77d5044 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 5 Dec 2024 10:58:56 +0800 Subject: [PATCH 59/99] Fix mem error at mndConfig.c. --- source/dnode/mnode/impl/src/mndConfig.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index f7065ddf09..d400f32d25 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -386,9 +386,7 @@ int32_t cfgUpdateItem(SConfigItem *pItem, SConfigObj *obj) { case CFG_DTYPE_STRING: { if (obj->str != NULL) { taosMemoryFree(pItem->str); - pItem->str = taosMemoryMalloc(strlen(obj->str) + 1); - tstrncpy(pItem->str, obj->str, strlen(obj->str)); - pItem->str[strlen(obj->str)] = 0; + pItem->str = taosStrdup(obj->str); } break; } From b7b185fa5fa12c2fb0d1e5fd614b834e3f415a7e Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 5 Dec 2024 16:21:05 +0800 Subject: [PATCH 60/99] Add check for disk id. --- include/os/osFile.h | 2 ++ include/util/tdef.h | 1 + source/common/src/tglobal.c | 71 +++++++++++++++++++++++++++++-------- source/os/src/osFile.c | 48 +++++++++++++++++++++++-- 4 files changed, 105 insertions(+), 17 deletions(-) diff --git a/include/os/osFile.h b/include/os/osFile.h index 1c397f3042..7bd99644d3 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -83,6 +83,8 @@ int32_t taosUnLockFile(TdFilePtr pFile); int32_t taosUmaskFile(int32_t maskVal); int32_t taosStatFile(const char *path, int64_t *size, int64_t *mtime, int64_t *atime); +int32_t taosGetFileDiskID(const char *path, int64_t *diskid); +bool taosCheckFileDiskID(const char *path, int64_t *actDiskID, int64_t expDiskID); int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno); int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int64_t *mtime); bool taosCheckExistFile(const char *pathname); diff --git a/include/util/tdef.h b/include/util/tdef.h index 246d1a293d..b46f472673 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -580,6 +580,7 @@ enum { ENCRYPT_KEY_STAT_UNKNOWN = 0, ENCRYPT_KEY_STAT_UNSET, ENCRYPT_KEY_STAT_SE typedef struct { char dir[TSDB_FILENAME_LEN]; int32_t level; + int64_t diskId; int32_t primary; int8_t disable; // disable create new file } SDiskCfg; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index a6bb249bc3..548b561d42 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1964,39 +1964,55 @@ int32_t cfgDeserialize(SArray *array, char *buf, bool isGlobal) { return TSDB_CODE_OUT_OF_MEMORY; } for (int i = 0; i < sz; i++) { - SConfigItem *item = (SConfigItem *)taosArrayGet(array, i); - cJSON *pItem = cJSON_GetObjectItem(configs, item->name); - if (pItem == NULL) { + SConfigItem *pItem = (SConfigItem *)taosArrayGet(array, i); + cJSON *pJson = cJSON_GetObjectItem(configs, pItem->name); + if (pJson == NULL) { continue; } - if (strstr(item->name, "syncLogBufferMemoryAllowed")) { - uDebug("syncLogBufferMemoryAllowed:%f", pItem->valuedouble); + if (strcasecmp(pItem->name, "dataDir") == 0) { + int sz = cJSON_GetArraySize(pJson); + cJSON *filed = NULL; + // check disk id for each dir + for (int j = 0; j < sz; j++) { + cJSON *diskCfgJson = cJSON_GetArrayItem(pJson, j); + filed = cJSON_GetObjectItem(diskCfgJson, "dir"); + char *dir = cJSON_GetStringValue(filed); + filed = cJSON_GetObjectItem(diskCfgJson, "disk_id"); + int64_t actDiskID = 0; + int64_t expDiskID = atoll(cJSON_GetStringValue(filed)); + if (!taosCheckFileDiskID(dir, &actDiskID, expDiskID)) { + uError("failed to check disk id for dir:%s, actDiskID:" PRId64 ", expDiskID:" PRId64, dir, actDiskID, + expDiskID); + return TSDB_CODE_FAILED; + } + } + continue; } - item->stype = CFG_STYPE_CFG_FILE; - switch (item->dtype) { + pItem->stype = CFG_STYPE_CFG_FILE; + switch (pItem->dtype) { { case CFG_DTYPE_NONE: break; case CFG_DTYPE_BOOL: - item->bval = cJSON_IsTrue(pItem); + pItem->bval = cJSON_IsTrue(pJson); break; case CFG_DTYPE_INT32: - item->i32 = pItem->valueint; + pItem->i32 = pJson->valueint; break; case CFG_DTYPE_INT64: - item->i64 = atoll(cJSON_GetStringValue(pItem)); + pItem->i64 = atoll(cJSON_GetStringValue(pJson)); break; case CFG_DTYPE_FLOAT: case CFG_DTYPE_DOUBLE: - item->fval = atoll(cJSON_GetStringValue(pItem)); + pItem->fval = atoll(cJSON_GetStringValue(pJson)); break; case CFG_DTYPE_STRING: case CFG_DTYPE_DIR: case CFG_DTYPE_LOCALE: case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: - taosMemoryFree(item->str); - item->str = taosStrdup(pItem->valuestring); + taosMemoryFree(pItem->str); + pItem->str = taosStrdup(pJson->valuestring); break; } } @@ -2127,7 +2143,6 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile TAOS_CHECK_GOTO(taosUpdateServerCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosSetServerCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosSetReleaseCfg(tsCfg), &lino, _exit); - TAOS_CHECK_GOTO(taosSetTfsCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosSetS3Cfg(tsCfg), &lino, _exit); } @@ -2820,6 +2835,34 @@ int32_t localConfigSerialize(SArray *array, char **serialized) { // string are used to prohibit the loss of precision for (int i = 0; i < sz; i++) { SConfigItem *item = (SConfigItem *)taosArrayGet(array, i); + if (strcasecmp(item->name, "dataDir") == 0) { + int32_t sz = taosArrayGetSize(item->array); + cJSON *dataDirs = cJSON_CreateArray(); + if (!cJSON_AddItemToObject(cField, item->name, dataDirs)) { + uError("failed to serialize global config since %s", tstrerror(terrno)); + } + for (int j = 0; j < sz; j++) { + SDiskCfg *disk = (SDiskCfg *)taosArrayGet(item->array, j); + cJSON *dataDir = cJSON_CreateObject(); + if (dataDir == NULL) goto _exit; + if (!cJSON_AddItemToArray(dataDirs, dataDir)) { + uError("failed to serialize global config since %s", tstrerror(terrno)); + } + if (cJSON_AddStringToObject(dataDir, "dir", disk->dir) == NULL) goto _exit; + if (cJSON_AddNumberToObject(dataDir, "level", disk->level) == NULL) goto _exit; + if (disk->diskId == 0) { + if (taosGetFileDiskID(disk->dir, &disk->diskId) != 0) { + uError("failed to get disk id for %s", disk->dir); + goto _exit; + } + } + (void)sprintf(buf, "%" PRId64, disk->diskId); + if (cJSON_AddStringToObject(dataDir, "disk_id", buf) == NULL) goto _exit; + if (cJSON_AddNumberToObject(dataDir, "primary", disk->primary) == NULL) goto _exit; + if (cJSON_AddNumberToObject(dataDir, "disable", disk->disable) == NULL) goto _exit; + } + continue; + } switch (item->dtype) { { case CFG_DTYPE_NONE: diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 5030eb1f67..27450b810f 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -198,7 +198,7 @@ _err: } TdFilePtr taosCreateFile(const char *path, int32_t tdFileOptions) { - if(path == NULL) { + if (path == NULL) { terrno = TSDB_CODE_INVALID_PARA; return NULL; } @@ -301,6 +301,48 @@ int32_t taosStatFile(const char *path, int64_t *size, int64_t *mtime, int64_t *a return 0; } + +int32_t taosGetFileDiskID(const char *path, int64_t *diskid) { + OS_PARAM_CHECK(path); +#ifdef WINDOWS + struct _stati64 fileStat; + int32_t code = _stati64(path, &fileStat); +#else + struct stat fileStat; + int32_t code = stat(path, &fileStat); +#endif + if (-1 == code) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } + + if (diskid != NULL) { + *diskid = fileStat.st_dev; + } + + return 0; +} + +bool taosCheckFileDiskID(const char *path, int64_t *actDiskID, int64_t expDiskID) { + OS_PARAM_CHECK(path); +#ifdef WINDOWS + struct _stati64 fileStat; + int32_t code = _stati64(path, &fileStat); +#else + struct stat fileStat; + int32_t code = stat(path, &fileStat); +#endif + if (-1 == code) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } + + if (actDiskID != NULL) { + *actDiskID = fileStat.st_dev; + } + return fileStat.st_dev == expDiskID; +} + int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { #ifdef WINDOWS if (pFile == NULL || pFile->hFile == NULL) { @@ -429,7 +471,7 @@ HANDLE taosOpenFileNotStream(const char *path, int32_t tdFileOptions) { } int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { - if (pFile == NULL || buf == NULL) { + if (pFile == NULL || buf == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -1572,7 +1614,7 @@ FILE *taosOpenCFile(const char *filename, const char *mode) { } int taosSeekCFile(FILE *file, int64_t offset, int whence) { - if(NULL == file) { + if (NULL == file) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } From 3a32980577bfc8f86a33ffe858ae3114f2f7fcef Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 5 Dec 2024 17:00:57 +0800 Subject: [PATCH 61/99] Fix some ci case errors. --- include/util/taoserror.h | 2 +- source/common/src/tglobal.c | 2 +- source/dnode/mnode/impl/test/profile/profile.cpp | 2 +- tests/system-test/0-others/persisit_config.py | 2 +- tests/system-test/0-others/test_show_disk_usage.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 4d57f13401..a01bbf66a6 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -582,7 +582,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_TDB_INVALID_CREATE_TB_MSG TAOS_DEF_ERROR_CODE(0, 0x060E) #define TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM TAOS_DEF_ERROR_CODE(0, 0x060F) #define TSDB_CODE_TDB_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0610) -#define TSDB_CODE_TDB_TABLE_RECONFIGURE TAOS_DEF_ERROR_CODE(0, 0x0611) +#define TSDB_CODE_TDB_TABLE_RECONFIGURE TAOS_DEF_ERROR_CODE(0, 0x0611) #define TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO TAOS_DEF_ERROR_CODE(0, 0x0612) #define TSDB_CODE_TDB_NO_AVAIL_DISK TAOS_DEF_ERROR_CODE(0, 0x0613) #define TSDB_CODE_TDB_MESSED_MSG TAOS_DEF_ERROR_CODE(0, 0x0614) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 548b561d42..9e35639ecf 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1981,7 +1981,7 @@ int32_t cfgDeserialize(SArray *array, char *buf, bool isGlobal) { int64_t actDiskID = 0; int64_t expDiskID = atoll(cJSON_GetStringValue(filed)); if (!taosCheckFileDiskID(dir, &actDiskID, expDiskID)) { - uError("failed to check disk id for dir:%s, actDiskID:" PRId64 ", expDiskID:" PRId64, dir, actDiskID, + uError("failed to check disk id for dir:%s, actDiskID%" PRId64 ", expDiskID%" PRId64, dir, actDiskID, expDiskID); return TSDB_CODE_FAILED; } diff --git a/source/dnode/mnode/impl/test/profile/profile.cpp b/source/dnode/mnode/impl/test/profile/profile.cpp index 0baff7d06e..68bd8cab40 100644 --- a/source/dnode/mnode/impl/test/profile/profile.cpp +++ b/source/dnode/mnode/impl/test/profile/profile.cpp @@ -94,7 +94,7 @@ TEST_F(MndTestProfile, 02_ConnectMsg_NotExistDB) { TEST_F(MndTestProfile, 03_ConnectMsg_Show) { test.SendShowReq(TSDB_MGMT_TABLE_CONNS, "perf_connections", ""); - EXPECT_EQ(test.GetShowRows(), 8); + EXPECT_EQ(test.GetShowRows(), 1); } TEST_F(MndTestProfile, 04_HeartBeatMsg) { diff --git a/tests/system-test/0-others/persisit_config.py b/tests/system-test/0-others/persisit_config.py index b7d437de3d..fa9f476f7d 100644 --- a/tests/system-test/0-others/persisit_config.py +++ b/tests/system-test/0-others/persisit_config.py @@ -98,7 +98,7 @@ class TDTestCase: def stop(self): tdSql.close() - # tdLog.success(f"{__file__} successfully executed") + tdLog.success(f"{__file__} successfully executed") tdCases.addLinux(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/test_show_disk_usage.py b/tests/system-test/0-others/test_show_disk_usage.py index abaecffec4..eb5bdf1aa7 100644 --- a/tests/system-test/0-others/test_show_disk_usage.py +++ b/tests/system-test/0-others/test_show_disk_usage.py @@ -191,7 +191,7 @@ class TDTestCase: self.value_check(icache, cacheRdbSize, 64) self.value_check(itableMeta,tableMetaSize, 64) self.value_check(itsdbSize, tsdbSize, 64) - self.value_check(iwal, walSize, 128) + self.value_check(iwal, walSize, 256) #if abs(icache - cacheRdbSize) > 12: # tdLog.error("cache_rdb size is not equal") From c653daf5dc41a519de203f7aafc80d48dda2aaf8 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Fri, 6 Dec 2024 11:37:45 +0800 Subject: [PATCH 62/99] Fix unit test. --- include/common/tglobal.h | 1 - source/common/src/tglobal.c | 5 - source/dnode/mgmt/test/sut/src/server.cpp | 5 +- source/dnode/mnode/impl/src/mndConfig.c | 73 ++++++++------- source/dnode/mnode/impl/src/mndDef.c | 2 +- tests/army/alter/test_alter_config.py | 106 +++++++++++++++++++++- tests/parallel_test/cases.task | 1 + 7 files changed, 145 insertions(+), 48 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 67bec995d2..7ae0f5caf1 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -54,7 +54,6 @@ extern uint16_t tsServerPort; extern int32_t tsVersion; extern int32_t tsForceReadConfig; extern int32_t tsdmConfigVersion; -extern int32_t tsmmConfigVersion; extern int32_t tsConfigInited; extern int32_t tsStatusInterval; extern int32_t tsNumOfSupportVnodes; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 9e35639ecf..a52edfa187 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -46,7 +46,6 @@ uint16_t tsServerPort = 6030; int32_t tsVersion = 30000000; int32_t tsForceReadConfig = 0; int32_t tsdmConfigVersion = -1; -int32_t tsmmConfigVersion = -1; int32_t tsConfigInited = 0; int32_t tsStatusInterval = 1; // second int32_t tsNumOfSupportVnodes = 256; @@ -2630,7 +2629,6 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { {"keepColumnName", &tsKeepColumnName}, {"logKeepDays", &tsLogKeepDays}, {"maxInsertBatchRows", &tsMaxInsertBatchRows}, - {"maxRetryWaitTime", &tsMaxRetryWaitTime}, {"minSlidingTime", &tsMinSlidingTime}, {"minIntervalTime", &tsMinIntervalTime}, {"numOfLogLines", &tsNumOfLogLines}, @@ -2641,13 +2639,10 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { {"queryNodeChunkSize", &tsQueryNodeChunkSize}, {"queryUseNodeAllocator", &tsQueryUseNodeAllocator}, {"smlDot2Underline", &tsSmlDot2Underline}, - {"shellActivityTimer", &tsShellActivityTimer}, {"useAdapter", &tsUseAdapter}, - {"experimental", &tsExperimental}, {"multiResultFunctionStarReturnTags", &tsMultiResultFunctionStarReturnTags}, {"maxTsmaCalcDelay", &tsMaxTsmaCalcDelay}, {"tsmaDataDeleteMark", &tsmaDataDeleteMark}, - {"safetyCheckLevel", &tsSafetyCheckLevel}, {"bypassFlag", &tsBypassFlag}}; if ((code = taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true)) != TSDB_CODE_SUCCESS) { diff --git a/source/dnode/mgmt/test/sut/src/server.cpp b/source/dnode/mgmt/test/sut/src/server.cpp index 52d697cb5d..e7dc45a0f8 100644 --- a/source/dnode/mgmt/test/sut/src/server.cpp +++ b/source/dnode/mgmt/test/sut/src/server.cpp @@ -17,10 +17,7 @@ void* serverLoop(void* param) { TestServer* server = (TestServer*)param; - - if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0){ - return NULL; - } + cfgInit(&tsCfg); if (dmInit() != 0) { return NULL; diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index d400f32d25..5f4dcd8278 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -32,7 +32,8 @@ enum CfgAlterType { static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pInMCfgReq, int32_t optLen, int32_t *pOutValue); static int32_t cfgUpdateItem(SConfigItem *pItem, SConfigObj *obj); -static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue, ECfgDataType dtype); +static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue, ECfgDataType dtype, + int32_t tsmmConfigVersion); static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq); static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessConfigReq(SRpcMsg *pReq); @@ -229,10 +230,18 @@ static int32_t mndProcessConfigReq(SRpcMsg *pReq) { mError("failed to deserialize config req, since %s", terrstr()); goto _OVER; } + + SConfigObj *vObj = sdbAcquire(pMnode->pSdb, SDB_CFG, "tsmmConfigVersion"); + if (vObj == NULL) { + mInfo("failed to acquire mnd config version, since %s", terrstr()); + goto _OVER; + } + SArray *diffArray = taosArrayInit(16, sizeof(SConfigItem)); SConfigRsp configRsp = {0}; configRsp.forceReadConfig = configReq.forceReadConfig; - configRsp.cver = tsmmConfigVersion; + + configRsp.cver = vObj->i32; if (configRsp.forceReadConfig) { // compare config array from configReq with current config array if (compareSConfigItemArrays(taosGetGlobalCfg(tsCfg), configReq.array, diffArray)) { @@ -242,7 +251,7 @@ static int32_t mndProcessConfigReq(SRpcMsg *pReq) { } } else { configRsp.array = taosGetGlobalCfg(tsCfg); - if (configReq.cver == tsmmConfigVersion) { + if (configReq.cver == vObj->i32) { configRsp.isVersionVerified = 1; } else { configRsp.array = taosGetGlobalCfg(tsCfg); @@ -316,11 +325,12 @@ int32_t mndInitReadCfg(SMnode *pMnode) { } SConfigObj *obj = sdbAcquire(pMnode->pSdb, SDB_CFG, "tsmmConfigVersion"); if (obj == NULL) { - mInfo("failed to acquire mnd config version, since %s", terrstr()); + code = mndInitWriteCfg(pMnode); + if (code != 0) { + mError("failed to init write cfg, since %s", terrstr()); + } + mInfo("failed to acquire mnd config version, try to rebuild it , since %s", terrstr()); goto _OVER; - } else { - tsmmConfigVersion = obj->i32; - sdbRelease(pMnode->pSdb, obj); } sz = taosArrayGetSize(taosGetGlobalCfg(tsCfg)); @@ -469,6 +479,11 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { int32_t lino = -1; SMnode *pMnode = pReq->info.node; SMCfgDnodeReq cfgReq = {0}; + SConfigObj *vObj = sdbAcquire(pMnode->pSdb, SDB_CFG, "tsmmConfigVersion"); + if (vObj == NULL) { + mInfo("failed to acquire mnd config version, since %s", terrstr()); + TAOS_RETURN(terrno); + } TAOS_CHECK_RETURN(tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq)); int8_t updateIpWhiteList = 0; mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value); @@ -521,21 +536,22 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { code = TSDB_CODE_CFG_NOT_FOUND; goto _err_out; } + +_send_req: if (pItem->category == CFG_CATEGORY_GLOBAL) { - TAOS_CHECK_GOTO(mndConfigUpdateTrans(pMnode, dcfgReq.config, dcfgReq.value, pItem->dtype), &lino, _err_out); + TAOS_CHECK_GOTO(mndConfigUpdateTrans(pMnode, dcfgReq.config, dcfgReq.value, pItem->dtype, ++vObj->i32), &lino, + _err_out); } -_send_req : + { // audit + char obj[50] = {0}; + (void)sprintf(obj, "%d", cfgReq.dnodeId); -{ // audit - char obj[50] = {0}; - (void)sprintf(obj, "%d", cfgReq.dnodeId); - - auditRecord(pReq, pMnode->clusterId, "alterDnode", obj, "", cfgReq.sql, cfgReq.sqlLen); -} + auditRecord(pReq, pMnode->clusterId, "alterDnode", obj, "", cfgReq.sql, cfgReq.sqlLen); + } tFreeSMCfgDnodeReq(&cfgReq); - dcfgReq.version = tsmmConfigVersion; + dcfgReq.version = vObj->i32; code = mndSendCfgDnodeReq(pMnode, cfgReq.dnodeId, &dcfgReq); // dont care suss or succ; @@ -577,25 +593,15 @@ _err: TAOS_RETURN(code); } -static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue, ECfgDataType dtype) { - int32_t code = -1; - int32_t lino = -1; - // SConfigObj *pVersion = sdbAcquire(pMnode->pSdb, SDB_CFG, "tsmmConfigVersion"); - // if (pVersion == NULL) { - // mError("failed to acquire tsmmConfigVersion while update config, since %s", terrstr()); - // code = terrno; - // goto _OVER; - // } - // pVersion->i32 = ++tsmmConfigVersion; - // SConfigObj *pObj = sdbAcquire(pMnode->pSdb, SDB_CFG, name); - // if (pObj == NULL) { - // mError("failed to acquire mnd config:%s while update config, since %s", name, terrstr()); - // code = terrno; - // goto _OVER; - // } +static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue, ECfgDataType dtype, + int32_t tsmmConfigVersion) { + int32_t code = -1; + int32_t lino = -1; SConfigObj pVersion = {0}, pObj = {0}; - pVersion.i32 = ++tsmmConfigVersion; + strncpy(pVersion.name, "tsmmConfigVersion", CFG_NAME_MAX_LEN); + pVersion.i32 = tsmmConfigVersion; + pVersion.dtype = CFG_DTYPE_INT32; pObj.dtype = dtype; strncpy(pObj.name, name, CFG_NAME_MAX_LEN); @@ -613,7 +619,6 @@ static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pVal code = 0; _OVER: if (code != 0) { - --tsmmConfigVersion; mError("failed to update config:%s to value:%s, since %s", name, pValue, tstrerror(code)); } mndTransDrop(pTrans); diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index f257991381..6d66addd6e 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -822,7 +822,7 @@ SConfigObj *mndInitConfigVersion() { } strncpy(pObj->name, "tsmmConfigVersion", CFG_NAME_MAX_LEN); pObj->dtype = CFG_DTYPE_INT32; - pObj->i32 = ++tsmmConfigVersion; + pObj->i32 = 0; return pObj; } diff --git a/tests/army/alter/test_alter_config.py b/tests/army/alter/test_alter_config.py index 7395268549..3f07ca4e06 100644 --- a/tests/army/alter/test_alter_config.py +++ b/tests/army/alter/test_alter_config.py @@ -37,6 +37,107 @@ class TDTestCase: self.configration_dic = { "cli": [ + { + "name": "asyncLog", + "value": 0, + "category": "local", + }, + { + "name": "enableQueryHb", + "value": 0, + "category": "local", + }, + { + "name": "keepColumnName", + "value": 1, + "category": "local", + }, + { + "name": "logKeepDays", + "value": 30, + "category": "local", + }, + { + "name": "maxInsertBatchRows", + "value": 2000000, + "category": "local", + }, + { + "name": "minSlidingTime", + "value": 10000, + "category": "local", + }, + { + "name": "minIntervalTime", + "value": 2, + "category": "local", + }, + { + "name": "numOfLogLines", + "value": 20000000, + "category": "local", + + }, + { + "name": "querySmaOptimize", + "value": 1, + "category": "local", + }, + { + "name": "queryPolicy", + "value": 3, + "category": "local", + }, + { + "name": "queryTableNotExistAsEmpty", + "value": 1, + "category": "local", + }, + { + "name": "queryPlannerTrace", + "value": 1, + "category": "local", + }, + { + "name": "queryNodeChunkSize", + "value": 16 * 1024, + "category": "local", + }, + { + "name": "queryUseNodeAllocator", + "value": 0, + "category": "local", + }, + { + "name": "smlDot2Underline", + "value": 0, + "category": "local", + }, + { + "name": "useAdapter", + "value": 1, + "category": "local", + }, + # { + # "name": "multiResultFunctionStarReturnTags", + # "value": 1, + # "category": "local", + # }, + { + "name": "maxTsmaCalcDelay", + "value": 1200, + "category": "local", + }, + { + "name": "tsmaDataDeleteMark", + "value": 1000 * 60 * 60 * 12, + "category": "local", + }, + { + "name": "bypassFlag", + "value": 4, + "category": "local", + } ], "svr": [ { @@ -528,18 +629,17 @@ class TDTestCase: tdLog.info("success to alter all configurations") tdLog.info("stop and restart taosd") - time.sleep(3) + sc.dnodeStopAll() sc.dnodeStart(1) sc.dnodeStart(2) sc.dnodeStart(3) - time.sleep(10) for key in self.configration_dic: if "cli" == key: for item in self.configration_dic[key]: actVal = self.cli_get_param_value(item["name"]) - assert str(actVal) == str(item["oldVal"]), f"item name: {item['name']}, Expected value: {item['value']}, actual value: {actVal}" + assert str(actVal) == str(item["value"]), f"item name: {item['name']}, Expected value: {item['value']}, actual value: {actVal}" elif "svr" == key: for item in self.configration_dic[key]: actVal = self.svr_get_param_value(item["name"]) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index b9dc38d41f..4b189573c1 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -41,6 +41,7 @@ ,,n,army,python3 ./test.py -f cmdline/fullopt.py ,,y,army,./pytest.sh python3 ./test.py -f query/show.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f alter/alterConfig.py -N 3 +,,y,army,./pytest.sh python3 ./test.py -f alter/test_alter_config.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f query/subquery/subqueryBugs.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f storage/oneStageComp.py -N 3 -L 3 -D 1 ,,y,army,./pytest.sh python3 ./test.py -f storage/compressBasic.py -N 3 From 36c7d0f146b8642692b580e06839aa539e82827e Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Fri, 6 Dec 2024 14:17:43 +0800 Subject: [PATCH 63/99] Fix ci cases. --- include/common/tglobal.h | 2 +- source/common/src/tglobal.c | 6 +++--- source/dnode/mnode/impl/src/mndConfig.c | 14 +++++++------- source/dnode/mnode/impl/test/profile/profile.cpp | 10 ++-------- 4 files changed, 13 insertions(+), 19 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 7ae0f5caf1..5f5805b4b9 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -160,7 +160,7 @@ extern bool tsEnableCrashReport; extern char *tsTelemUri; extern char *tsClientCrashReportUri; extern char *tsSvrCrashReportUri; -extern int8_t tsSafetyCheckLevel; +extern int32_t tsSafetyCheckLevel; enum { TSDB_SAFETY_CHECK_LEVELL_NEVER = 0, TSDB_SAFETY_CHECK_LEVELL_NORMAL = 1, diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index a52edfa187..2cffcf100e 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -145,9 +145,9 @@ bool tsEnableCrashReport = false; #else bool tsEnableCrashReport = true; #endif -char *tsClientCrashReportUri = "/ccrashreport"; -char *tsSvrCrashReportUri = "/dcrashreport"; -int8_t tsSafetyCheckLevel = TSDB_SAFETY_CHECK_LEVELL_NORMAL; +char *tsClientCrashReportUri = "/ccrashreport"; +char *tsSvrCrashReportUri = "/dcrashreport"; +int32_t tsSafetyCheckLevel = TSDB_SAFETY_CHECK_LEVELL_NORMAL; // schemaless bool tsSmlDot2Underline = true; diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index 5f4dcd8278..a2c439cfed 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -536,18 +536,18 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { code = TSDB_CODE_CFG_NOT_FOUND; goto _err_out; } - -_send_req: if (pItem->category == CFG_CATEGORY_GLOBAL) { TAOS_CHECK_GOTO(mndConfigUpdateTrans(pMnode, dcfgReq.config, dcfgReq.value, pItem->dtype, ++vObj->i32), &lino, _err_out); } - { // audit - char obj[50] = {0}; - (void)sprintf(obj, "%d", cfgReq.dnodeId); +_send_req : - auditRecord(pReq, pMnode->clusterId, "alterDnode", obj, "", cfgReq.sql, cfgReq.sqlLen); - } +{ // audit + char obj[50] = {0}; + (void)sprintf(obj, "%d", cfgReq.dnodeId); + + auditRecord(pReq, pMnode->clusterId, "alterDnode", obj, "", cfgReq.sql, cfgReq.sqlLen); +} tFreeSMCfgDnodeReq(&cfgReq); diff --git a/source/dnode/mnode/impl/test/profile/profile.cpp b/source/dnode/mnode/impl/test/profile/profile.cpp index 68bd8cab40..15b90463b6 100644 --- a/source/dnode/mnode/impl/test/profile/profile.cpp +++ b/source/dnode/mnode/impl/test/profile/profile.cpp @@ -57,14 +57,8 @@ TEST_F(MndTestProfile, 01_ConnectMsg) { EXPECT_NE(connectRsp.connId, 0); EXPECT_EQ(connectRsp.superUser, 1); - EXPECT_EQ(connectRsp.epSet.inUse, 0); - EXPECT_EQ(connectRsp.epSet.numOfEps, 1); - EXPECT_EQ(connectRsp.epSet.eps[0].port, 6030); - char defaultFqdn[TSDB_FQDN_LEN] = {0}; - if (taosGetFqdn(defaultFqdn) != 0) { - (void)strcpy(defaultFqdn, "localhost"); - } - EXPECT_STREQ(connectRsp.epSet.eps[0].fqdn, defaultFqdn); + EXPECT_EQ(connectRsp.epSet.eps[0].port, 9031); + EXPECT_STREQ(connectRsp.epSet.eps[0].fqdn, "localhost"); connId = connectRsp.connId; } From ef16c330882c7e3cc5a7546158b6bec9e9fa6b4b Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Fri, 6 Dec 2024 16:48:15 +0800 Subject: [PATCH 64/99] Add more cases. --- source/common/src/tglobal.c | 8 +- tests/army/alter/alterAllConfig.py | 202 ----------------------------- tests/army/alter/alterConfig.py | 6 + 3 files changed, 10 insertions(+), 206 deletions(-) delete mode 100644 tests/army/alter/alterAllConfig.py diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 2cffcf100e..9e5d0290cd 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -716,7 +716,7 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddLocale(pCfg, "locale", tsLocale, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddCharset(pCfg, "charset", tsCharset, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN( - cfgAddBool(pCfg, "enableCoreFile", tsEnableCoreFile, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); + cfgAddBool(pCfg, "enableCoreFile", tsEnableCoreFile, CFG_SCOPE_BOTH, CFG_DYN_BOTH_LAZY, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN( cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_BOTH_LAZY, CFG_CATEGORY_LOCAL)); @@ -791,7 +791,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddString(pCfg, "encryptAlgorithm", tsEncryptAlgorithm, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "encryptScope", tsEncryptScope, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL)); @@ -835,7 +835,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitorLogProtocol", tsMonitorLogProtocol, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitorForceV2", tsMonitorForceV2, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); @@ -865,7 +865,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlBatchDropNum", tsTtlBatchDropNum, 0, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "ttlChangeOnWrite", tsTtlChangeOnWrite, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "ttlChangeOnWrite", tsTtlChangeOnWrite, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlFlushThreshold", tsTtlFlushThreshold, -1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3MigrateIntervalSec", tsS3MigrateIntervalSec, 600, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); diff --git a/tests/army/alter/alterAllConfig.py b/tests/army/alter/alterAllConfig.py deleted file mode 100644 index f5de2232ab..0000000000 --- a/tests/army/alter/alterAllConfig.py +++ /dev/null @@ -1,202 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - -import sys -import time - -import taos -import frame -import frame.etool - -from frame.log import * -from frame.cases import * -from frame.sql import * -from frame.caseBase import * -from frame.epath import * -from frame.srvCtl import * -from frame import * - -class ConfigParam: - def __init__(self, name, default_value, value_range, scope,category): - self.name = name - self.default_value = default_value - self.value_range = value_range - self.scope = scope - self.category = category - - - -class TDTestCase(TBase): - def alterCachemodel(self): - """Add test case for altering cache model(TS-5390) - """ - # drop database if exists - tdSql.execute(f"drop database if exists db3590;") - # create database with cachemodel 'none' - tdSql.execute(f"create database db3590 vgroups 1 replica 1 cachemodel 'none';") - tdSql.execute("use db3590;") - tdSql.execute("create table ntb1 (ts timestamp, ival1 int);") - tdSql.execute("insert into ntb1 values(now, 1);") - tdSql.execute("flush database db3590;") - # alter table schema - tdSql.execute("alter table ntb1 add column ival2 int;") - tdSql.execute("insert into ntb1 values(now, 2, NULL);") - # alter table schema again - tdSql.execute("alter table ntb1 add column ival3 int;") - # alter database with cachemodel 'both‘ - tdSql.execute("alter database db3590 cachemodel 'both';") - # wait for cachemodel avaiable - time.sleep(5) - tdSql.query("select last(*) from ntb1;") - - def alterTtlConfig(self): - """Add test case for altering ttl config - """ - db_name = "db" - tdSql.execute(f"create database {db_name};") - tdSql.execute(f"use {db_name};") - tdSql.execute("create table t1 (ts timestamp, a int);") - - ttl_min_value = 0 - ttl_max_value = 2147483647 - # verify ttl min value - tdSql.execute(f"alter table t1 ttl {ttl_min_value}") - tdSql.query("select `ttl` from information_schema.ins_tables where table_name='t1' and db_name='db';") - tdSql.checkData(0, 0, ttl_min_value) - # verify ttl max value - tdSql.execute(f"alter table t1 ttl {ttl_max_value}") - tdSql.query("select `ttl` from information_schema.ins_tables where table_name='t1' and db_name='db';") - tdSql.checkData(0, 0, ttl_max_value) - # verify abnormal ttl value - tdSql.error(f"alter table t1 ttl {ttl_max_value + 1}", expectErrInfo="Value out of range") - tdSql.error(f"alter table t1 ttl {ttl_min_value - 1}", expectErrInfo="syntax error") - - # TS-5291 - tdSql.execute("create database db5291 vgroups 1;") - tdSql.execute("use db5291;") - tdSql.execute("create table ttltb1(ts timestamp, ival int) ttl 1;") - tdSql.execute("create table ttltb2(ts timestamp, ival int) ttl 1;") - tdSql.execute("drop table ttltb1;") - tdSql.execute("flush database db5291;") - tdSql.execute("drop table ttltb2;") - # restart taosd - sc.dnodeStopAll() - sc.dnodeStart(1) - sc.dnodeStart(2) - sc.dnodeStart(3) - tdSql.execute("flush database db5291;") - - def alterSupportVnodes(self): - tdLog.info(f"test function of altering supportVnodes") - - tdSql.execute("alter dnode 1 'supportVnodes' '128'") - time.sleep(1) - tdSql.query('show dnodes') - tdSql.checkData(0, 3, "128") - - tdSql.execute("alter dnode 1 'supportVnodes' '64'") - time.sleep(1) - tdSql.query('show dnodes') - tdSql.checkData(0, 3, "64") - - def checkKeyValue(self, res, key, value, ikey = 0, ival = 1): - result = False - for row in res: - if row[ikey] == key: - if row[ival] != value: - raise Exception(f"key:{key} value:{row[ival]} != {value}") - else: - tdLog.info(f"key:{key} value:{row[ival]} == {value}") - result = True - break - if not result: - raise Exception(f"key:{key} not found") - - def checkRows(self, sql, nExpect, nRetry): - for i in range(nRetry): - res = tdSql.getResult(sql) - if len(res) == nExpect: - break - time.sleep(1) - if len(res) != nExpect: - raise Exception(f"rows:{len(res)} != {nExpect}") - - def alterBypassFlag(self): - """Add test case for altering bypassFlag(TD-32907) - """ - tdSql.execute(f"drop database if exists db") - tdSql.execute(f"create database db") - tdSql.execute("use db") - self.checkKeyValue(tdSql.getResult("show local variables;"), "bypassFlag", "0") - self.checkKeyValue(tdSql.getResult("show dnode 1 variables like 'bypassFlag'"), "bypassFlag", "0", 1, 2) - tdSql.execute("alter local 'bypassFlag 1'") - self.checkKeyValue(tdSql.getResult("show local variables;"), "bypassFlag", "1") - self.checkKeyValue(tdSql.getResult("show dnode 1 variables like 'bypassFlag'"), "bypassFlag", "0", 1, 2) - tdSql.execute("create table stb0(ts timestamp, c0 int) tags(t0 int)") - tdSql.execute("create table ctb0 using stb0 tags(0)") - tdSql.execute("insert into ctb0 values(now, 1)") - tdSql.query("select * from stb0") - tdSql.checkRows(0) - tdSql.execute("alter local 'bypassFlag 0'") - tdSql.execute("alter all dnodes 'bypassFlag 2'") - self.checkKeyValue(tdSql.getResult("show local variables"), "bypassFlag", "0") - self.checkKeyValue(tdSql.getResult("show dnode 1 variables like 'bypassFlag'"), "bypassFlag", "2", 1, 2) - tdSql.execute("insert into ctb0 values(now, 2)") - tdSql.query("select * from stb0") - tdSql.checkRows(0) - tdSql.execute("alter all dnodes 'bypassFlag 4'") - self.checkKeyValue(tdSql.getResult("show dnode 1 variables like 'bypassFlag'"), "bypassFlag", "4", 1, 2) - tdSql.execute("insert into ctb0 values(now, 4)") - tdSql.execute("insert into ctb1 using stb0 tags(1) values(now, 10)") - tdSql.query("select * from stb0") - tdSql.checkRows(0) - tdSql.query("show db.tables") - tdSql.checkRows(2) - tdSql.execute("alter all dnodes 'bypassFlag 8'") - self.checkKeyValue(tdSql.getResult("show dnode 1 variables like 'bypassFlag'"), "bypassFlag", "8", 1, 2) - tdSql.execute("insert into ctb0 values(now, 8)") - tdSql.execute("insert into ctb1 values(now, 18)") - tdSql.query("select * from stb0") - tdSql.checkRows(2) - tdSql.execute("flush database db") - self.checkRows("select * from stb0", 0, 10) - tdSql.execute("alter all dnodes 'bypassFlag 0'") - self.checkKeyValue(tdSql.getResult("show local variables"), "bypassFlag", "0") - self.checkKeyValue(tdSql.getResult("show dnode 1 variables like 'bypassFlag'"), "bypassFlag", "0", 1, 2) - tdSql.execute("insert into ctb0 values(now, 80)") - tdSql.execute("insert into ctb1 values(now, 180)") - tdSql.query("select * from stb0") - tdSql.checkRows(2) - tdSql.execute("flush database db") - for i in range(5): - self.checkRows("select * from stb0", 2, 1) - time.sleep(1) - - # run - def run(self): - tdLog.debug(f"start to excute {__file__}") - - # TS-4721 - self.alterSupportVnodes() - # TS-5191 - self.alterTtlConfig() - # TS-5390 - self.alterCachemodel() - # TD-32907 - self.alterBypassFlag() - - tdLog.success(f"{__file__} successfully executed") - - -tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/alter/alterConfig.py b/tests/army/alter/alterConfig.py index 6a22dd014f..3fd0a51576 100644 --- a/tests/army/alter/alterConfig.py +++ b/tests/army/alter/alterConfig.py @@ -173,6 +173,10 @@ class TDTestCase(TBase): self.checkRows("select * from stb0", 2, 1) time.sleep(1) + def alter_err_case(self): + tdSql.error(f"alter local 'audit 0'",expectErrInfo="Config not found") + tdSql.error(f"alter dnode 1 'audit 1'",expectErrInfo="Invalid config option") + # run def run(self): tdLog.debug(f"start to excute {__file__}") @@ -185,6 +189,8 @@ class TDTestCase(TBase): self.alterCachemodel() # TD-32907 self.alterBypassFlag() + # TS-5007 + self.alter_err_case() tdLog.success(f"{__file__} successfully executed") From f79c2333b29fa74f5ae4cae5c933bc5a9d8dea5a Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Fri, 6 Dec 2024 17:34:09 +0800 Subject: [PATCH 65/99] Fix mnode = 3 ,config lost. --- source/dnode/mnode/impl/src/mndConfig.c | 85 +++++++++++++++++++++++-- 1 file changed, 79 insertions(+), 6 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index a2c439cfed..f0282bd6ab 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -39,6 +39,8 @@ static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessConfigReq(SRpcMsg *pReq); static int32_t mndInitWriteCfg(SMnode *pMnode); static int32_t mndInitReadCfg(SMnode *pMnode); +static int32_t initConfigArrayFromSdb(SMnode *pMnode, SArray *array); +static void cfgArrayCleanUp(SArray *array); int32_t mndSetCreateConfigCommitLogs(STrans *pTrans, SConfigObj *obj); @@ -224,6 +226,7 @@ static int32_t mndProcessConfigReq(SRpcMsg *pReq) { SConfigReq configReq = {0}; SDnodeObj *pDnode = NULL; int32_t code = -1; + SArray *array = NULL; code = tDeserializeSConfigReq(pReq->pCont, pReq->contLen, &configReq); if (code != 0) { @@ -237,24 +240,24 @@ static int32_t mndProcessConfigReq(SRpcMsg *pReq) { goto _OVER; } - SArray *diffArray = taosArrayInit(16, sizeof(SConfigItem)); + array = taosArrayInit(16, sizeof(SConfigItem)); SConfigRsp configRsp = {0}; configRsp.forceReadConfig = configReq.forceReadConfig; configRsp.cver = vObj->i32; if (configRsp.forceReadConfig) { // compare config array from configReq with current config array - if (compareSConfigItemArrays(taosGetGlobalCfg(tsCfg), configReq.array, diffArray)) { - configRsp.array = diffArray; + if (compareSConfigItemArrays(taosGetGlobalCfg(tsCfg), configReq.array, array)) { + configRsp.array = array; } else { configRsp.isConifgVerified = 1; } } else { - configRsp.array = taosGetGlobalCfg(tsCfg); if (configReq.cver == vObj->i32) { configRsp.isVersionVerified = 1; } else { - configRsp.array = taosGetGlobalCfg(tsCfg); + initConfigArrayFromSdb(pMnode, array); + configRsp.array = array; } } @@ -273,7 +276,7 @@ static int32_t mndProcessConfigReq(SRpcMsg *pReq) { pReq->info.rsp = pHead; _OVER: - taosArrayDestroy(diffArray); + cfgArrayCleanUp(array); mndReleaseDnode(pMnode, pDnode); return TSDB_CODE_SUCCESS; } @@ -623,4 +626,74 @@ _OVER: } mndTransDrop(pTrans); return code; +} + +static int32_t initConfigArrayFromSdb(SMnode *pMnode, SArray *array) { + int32_t code = 0; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + while (1) { + SConfigObj *obj = NULL; + pIter = sdbFetch(pSdb, SDB_CFG, pIter, (void **)&obj); + if (pIter == NULL) break; + if (obj == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + break; + } + if (strcasecmp(obj->name, "tsmmConfigVersion") == 0) { + continue; + } + SConfigItem item = {0}; + item.dtype = obj->dtype; + item.name = taosStrdup(obj->name); + switch (obj->dtype) { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + item.bval = obj->bval; + break; + case CFG_DTYPE_INT32: + item.i32 = obj->i32; + break; + case CFG_DTYPE_INT64: + item.i64 = obj->i64; + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + item.fval = obj->fval; + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + item.str = taosStrdup(obj->str); + break; + } + if (taosArrayPush(array, &item) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + sdbRelease(pSdb, obj); + break; + } + sdbRelease(pSdb, obj); + } + return code; +} + +static void cfgArrayCleanUp(SArray *array) { + if (array == NULL) { + return; + } + + int32_t sz = taosArrayGetSize(array); + for (int32_t i = 0; i < sz; ++i) { + SConfigItem *item = taosArrayGet(array, i); + if (item->dtype == CFG_DTYPE_STRING || item->dtype == CFG_DTYPE_DIR || item->dtype == CFG_DTYPE_LOCALE || + item->dtype == CFG_DTYPE_CHARSET || item->dtype == CFG_DTYPE_TIMEZONE) { + taosMemoryFreeClear(item->str); + } + taosMemoryFreeClear(item->name); + } + + taosArrayDestroy(array); } \ No newline at end of file From 3343f0d44e9f165dc2b94a8091643126a5aa9608 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Fri, 6 Dec 2024 17:44:39 +0800 Subject: [PATCH 66/99] Add sleep time at case. --- tests/army/alter/test_alter_config.py | 1 + tests/parallel_test/cases.task | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/army/alter/test_alter_config.py b/tests/army/alter/test_alter_config.py index 3f07ca4e06..52072dae5c 100644 --- a/tests/army/alter/test_alter_config.py +++ b/tests/army/alter/test_alter_config.py @@ -634,6 +634,7 @@ class TDTestCase: sc.dnodeStart(1) sc.dnodeStart(2) sc.dnodeStart(3) + sleep(10) for key in self.configration_dic: if "cli" == key: diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 4b189573c1..4163439305 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -42,6 +42,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f query/show.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f alter/alterConfig.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f alter/test_alter_config.py -N 3 +,,y,army,./pytest.sh python3 ./test.py -f alter/test_alter_config.py -N 3 -M 3 ,,y,army,./pytest.sh python3 ./test.py -f query/subquery/subqueryBugs.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f storage/oneStageComp.py -N 3 -L 3 -D 1 ,,y,army,./pytest.sh python3 ./test.py -f storage/compressBasic.py -N 3 From d8c5c65f7417896b2d7a12dcb6c3ed0f8df286b5 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Fri, 6 Dec 2024 18:17:16 +0800 Subject: [PATCH 67/99] fix recv return code . --- source/dnode/mnode/impl/src/mndConfig.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index f0282bd6ab..b283c0f221 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -225,7 +225,7 @@ static int32_t mndProcessConfigReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SConfigReq configReq = {0}; SDnodeObj *pDnode = NULL; - int32_t code = -1; + int32_t code = TSDB_CODE_SUCCESS; SArray *array = NULL; code = tDeserializeSConfigReq(pReq->pCont, pReq->contLen, &configReq); @@ -256,7 +256,11 @@ static int32_t mndProcessConfigReq(SRpcMsg *pReq) { if (configReq.cver == vObj->i32) { configRsp.isVersionVerified = 1; } else { - initConfigArrayFromSdb(pMnode, array); + code = initConfigArrayFromSdb(pMnode, array); + if (code != 0) { + mError("failed to init config array from sdb, since %s", terrstr()); + goto _OVER; + } configRsp.array = array; } } From 7cb2de940a6512d60893420c670c508a68ea69ba Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 9 Dec 2024 15:35:20 +0800 Subject: [PATCH 68/99] Add docs. --- .../en/14-reference/01-components/01-taosd.md | 1 - .../zh/14-reference/01-components/01-taosd.md | 380 +++++++++--------- .../zh/14-reference/01-components/02-taosc.md | 163 ++++---- docs/zh/14-reference/03-taos-sql/21-node.md | 53 +-- source/common/src/tglobal.c | 60 ++- 5 files changed, 316 insertions(+), 341 deletions(-) diff --git a/docs/en/14-reference/01-components/01-taosd.md b/docs/en/14-reference/01-components/01-taosd.md index 436f7c522b..5dcf5f91ae 100644 --- a/docs/en/14-reference/01-components/01-taosd.md +++ b/docs/en/14-reference/01-components/01-taosd.md @@ -71,7 +71,6 @@ After modifying configuration file parameters, you need to restart the *taosd* s |------------------------|----------|-| |countAlwaysReturnValue | |Whether count/hyperloglog functions return a value when input data is empty or NULL; 0: return empty row, 1: return; default value 1; When this parameter is set to 1, if the query contains an INTERVAL clause or the query uses TSMA, and the corresponding group or window has empty or NULL data, the corresponding group or window will not return a query result; Note that this parameter should be consistent between client and server| |tagFilterCache | |Whether to cache tag filter results| -|maxNumOfDistinctRes | |Maximum number of distinct results allowed to return, default value 100,000, maximum allowed value 100 million| |queryBufferSize | |Not effective yet| |queryRspPolicy | |Query response strategy| |filterScalarMode | |Force scalar filter mode, 0: off; 1: on, default value 0| diff --git a/docs/zh/14-reference/01-components/01-taosd.md b/docs/zh/14-reference/01-components/01-taosd.md index 435a89c845..c70393faa3 100644 --- a/docs/zh/14-reference/01-components/01-taosd.md +++ b/docs/zh/14-reference/01-components/01-taosd.md @@ -26,65 +26,64 @@ taosd 命令行参数如下 ::: ### 连接相关 -|参数名称|支持版本|参数含义| -|-----------------------|-------------------------|------------| -|firstEp | |taosd 启动时，主动连接的集群中首个 dnode 的 end point，默认值 localhost:6030| -|secondEp | |taosd 启动时，如果 firstEp 连接不上，尝试连接集群中第二个 dnode 的 endpoint，无默认值| -|fqdn | |taosd 监听的服务地址，默认为所在服务器上配置的第一个 hostname| -|serverPort | |taosd 监听的端口，默认值 6030| -|compressMsgSize | |是否对 RPC 消息进行压缩；-1：所有消息都不压缩；0：所有消息都压缩；N (N>0)：只有大于 N 个字节的消息才压缩；默认值 -1| -|shellActivityTimer | |客户端向 mnode 发送心跳的时长，单位为秒，取值范围 1-120，默认值 3 | -|numOfRpcSessions | |RPC 支持的最大连接数，取值范围 100-100000，默认值 30000| -|numOfRpcThreads | |RPC 收发数据线程数目，取值范围1-50,默认值为 CPU 核数的一半| -|numOfTaskQueueThreads | |客户端处理 RPC 消息的线程数取值, 范围4-16,默认值为 CPU 核数的一半| -|rpcQueueMemoryAllowed | |dnode允许的已经收到的RPC消息占用的内存最大值，单位 bytes，取值范围 104857600-INT64_MAX，默认值为服务器内存的 1/10 | -|resolveFQDNRetryTime | 3.x 之后取消 |FQDN 解析失败时的重试次数| -|timeToGetAvailableConn | 3.3.4.x之后取消 |获得可用连接的最长等待时间，取值范围 10-50000000，单位为毫秒，默认值 500000| -|maxShellConns | 3.x 后取消 |允许创建的最大链接数| -|maxRetryWaitTime | |重连最大超时时间, 默认值是 10s| -|shareConnLimit |3.3.4.0 新增 |一个链接可以共享的请求的数目，取值范围 1-512，默认值 10| -|readTimeout |3.3.4.0 新增 |单个请求最小超时时间，取值范围 64-604800，单位为秒，默认值 900| +|参数名称|支持版本|动态修改|参数含义| +|-----------------------|-------------------------|-------------------------|------------| +|firstEp | |不支持动态修改 |taosd 启动时，主动连接的集群中首个 dnode 的 end point，默认值 localhost:6030| +|secondEp | |不支持动态修改 |taosd 启动时，如果 firstEp 连接不上，尝试连接集群中第二个 dnode 的 endpoint，无默认值| +|fqdn | |不支持动态修改 |taosd 监听的服务地址，默认为所在服务器上配置的第一个 hostname| +|serverPort | |不支持动态修改 |taosd 监听的端口，默认值 6030| +|compressMsgSize | |支持动态修改 重启生效 |是否对 RPC 消息进行压缩；-1：所有消息都不压缩；0：所有消息都压缩；N (N>0)：只有大于 N 个字节的消息才压缩；默认值 -1| +|shellActivityTimer | |支持动态修改 立即生效 |客户端向 mnode 发送心跳的时长，单位为秒，取值范围 1-120，默认值 3 | +|numOfRpcSessions | |支持动态修改 重启生效 |RPC 支持的最大连接数，取值范围 100-100000，默认值 30000| +|numOfRpcThreads | |支持动态修改 重启生效 |RPC 收发数据线程数目，取值范围1-50,默认值为 CPU 核数的一半| +|numOfTaskQueueThreads | |支持动态修改 重启生效 |客户端处理 RPC 消息的线程数取值, 范围4-16,默认值为 CPU 核数的一半| +|rpcQueueMemoryAllowed | |支持动态修改 立即生效 |dnode允许的已经收到的RPC消息占用的内存最大值，单位 bytes，取值范围 104857600-INT64_MAX，默认值为服务器内存的 1/10 | +|resolveFQDNRetryTime | 3.x 之后取消 |不支持动态修改 |FQDN 解析失败时的重试次数| +|timeToGetAvailableConn | 3.3.4.x之后取消 |支持动态修改 重启生效 |获得可用连接的最长等待时间，取值范围 10-50000000，单位为毫秒，默认值 500000| +|maxShellConns | 3.x 后取消 |支持动态修改 重启生效 |允许创建的最大链接数| +|maxRetryWaitTime | |支持动态修改 重启生效 |重连最大超时时间, 默认值是 10s| +|shareConnLimit |3.3.4.0 新增 |支持动态修改 重启生效 |一个链接可以共享的请求的数目，取值范围 1-512，默认值 10| +|readTimeout |3.3.4.0 新增 |支持动态修改 重启生效 |单个请求最小超时时间，取值范围 64-604800，单位为秒，默认值 900| ### 监控相关 -|参数名称|支持版本|参数含义| -|-----------------------|----------|-| -|monitor | |是否收集监控数据并上报，0：关闭；1:打开；默认值 0| -|monitorFqdn | |taosKeeper 服务所在服务器的 FQDN，默认值 无| -|monitorPort | |taosKeeper 服务所监听的端口号，默认值 6043| -|monitorInterval | |监控数据库记录系统参数（CPU/内存）的时间间隔，单位是秒，取值范围 1-200000 ，默认值 30| -|monitorMaxLogs | |缓存的待上报日志条数| -|monitorComp | |是否采用压缩方式上报监控日志时| -|monitorLogProtocol | |是否打印监控日志| -|monitorForceV2 | |是否使用 V2 版本协议上报| -|telemetryReporting | |是否上传 telemetry，0：不上传，1：上传，默认值 1| -|telemetryServer | |telemetry 服务器地址| -|telemetryPort | |telemetry 服务器端口编号| -|telemetryInterval | |telemetry 上传时间间隔，单位为秒，默认 43200| -|crashReporting | |是否上传 crash 信息；0：不上传，1：上传；默认值 1| +|参数名称|支持版本|动态修改|参数含义| +|-----------------------|----------|-------------------------|-| +|monitor | |支持动态修改 立即生效 |是否收集监控数据并上报，0：关闭；1:打开；默认值 0| +|monitorFqdn | |支持动态修改 重启生效 |taosKeeper 服务所在服务器的 FQDN，默认值 无| +|monitorPort | |支持动态修改 重启生效 |taosKeeper 服务所监听的端口号，默认值 6043| +|monitorInterval | |支持动态修改 立即生效 |监控数据库记录系统参数（CPU/内存）的时间间隔，单位是秒，取值范围 1-200000 ，默认值 30| +|monitorMaxLogs | |支持动态修改 立即生效 |缓存的待上报日志条数| +|monitorComp | |支持动态修改 重启生效 |是否采用压缩方式上报监控日志时| +|monitorLogProtocol | |支持动态修改 立即生效 |是否打印监控日志| +|monitorForceV2 | |支持动态修改 立即生效 |是否使用 V2 版本协议上报| +|telemetryReporting | |支持动态修改 立即生效 |是否上传 telemetry，0：不上传，1：上传，默认值 1| +|telemetryServer | |不支持动态修改 |telemetry 服务器地址| +|telemetryPort | |不支持动态修改 |telemetry 服务器端口编号| +|telemetryInterval | |支持动态修改 立即生效 |telemetry 上传时间间隔，单位为秒，默认 43200| +|crashReporting | |支持动态修改 立即生效 |是否上传 crash 信息；0：不上传，1：上传；默认值 1| ### 查询相关 -|参数名称|支持版本|参数含义| -|------------------------|----------|-| -|countAlwaysReturnValue | |count/hyperloglog 函数在输入数据为空或者 NULL 的情况下是否返回值；0：返回空行，1：返回；默认值 1；该参数设置为 1 时，如果查询中含有 INTERVAL 子句或者该查询使用了 TSMA 时，且相应的组或窗口内数据为空或者 NULL，对应的组或窗口将不返回查询结果；注意此参数客户端和服务端值应保持一致| -|tagFilterCache | |是否缓存标签过滤结果| -|maxNumOfDistinctRes | |允许返回的 distinct 结果最大行数，默认值 10 万，最大允许值 1 亿| -|queryBufferSize | |暂不生效| -|queryRspPolicy | |查询响应策略| -|filterScalarMode | |强制使用标量过滤模式，0：关闭；1：开启，默认值 0| -|queryPlannerTrace | |内部参数，查询计划是否输出详细日志| -|queryNodeChunkSize | |内部参数，查询计划的块大小| -|queryUseNodeAllocator | |内部参数，查询计划的分配方法| -|queryMaxConcurrentTables| |内部参数，查询计划的并发数目| -|queryRsmaTolerance | |内部参数，用于判定查询哪一级 rsma 数据时的容忍时间，单位为毫秒| -|enableQueryHb | |内部参数，是否发送查询心跳消息| -|pqSortMemThreshold | |内部参数，排序使用的内存阈值| +|参数名称|支持版本|动态修改|参数含义| +|------------------------|----------|-------------------------|-| +|countAlwaysReturnValue | |支持动态修改 立即生效 |count/hyperloglog 函数在输入数据为空或者 NULL 的情况下是否返回值；0：返回空行，1：返回；默认值 1；该参数设置为 1 时，如果查询中含有 INTERVAL 子句或者该查询使用了 TSMA 时，且相应的组或窗口内数据为空或者 NULL，对应的组或窗口将不返回查询结果；注意此参数客户端和服务端值应保持一致| +|tagFilterCache | |不支持动态修改 |是否缓存标签过滤结果| +|queryBufferSize | |支持动态修改 重启生效 |暂不生效| +|queryRspPolicy | |支持动态修改 立即生效 |查询响应策略| +|filterScalarMode | |不支持动态修改 |强制使用标量过滤模式，0：关闭；1：开启，默认值 0| +|queryPlannerTrace | |支持动态修改 立即生效 |内部参数，查询计划是否输出详细日志| +|queryNodeChunkSize | |支持动态修改 立即生效 |内部参数，查询计划的块大小| +|queryUseNodeAllocator | |支持动态修改 立即生效 |内部参数，查询计划的分配方法| +|queryMaxConcurrentTables| |不支持动态修改 |内部参数，查询计划的并发数目| +|queryRsmaTolerance | |不支持动态修改 |内部参数，用于判定查询哪一级 rsma 数据时的容忍时间，单位为毫秒| +|enableQueryHb | |支持动态修改 立即生效 |内部参数，是否发送查询心跳消息| +|pqSortMemThreshold | |不支持动态修改 |内部参数，排序使用的内存阈值| ### 区域相关 -|参数名称|支持版本|参数含义| -|-----------------|----------|-| -|timezone | |时区；缺省从系统中动态获取当前的时区设置| -|locale | |系统区位信息及编码格式，缺省从系统中获取| -|charset | |字符集编码，缺省从系统中获取| +|参数名称|支持版本|动态修改|参数含义| +|-----------------|----------|-------------------------|-| +|timezone | |不支持动态修改 |时区；缺省从系统中动态获取当前的时区设置| +|locale | |不支持动态修改 |系统区位信息及编码格式，缺省从系统中获取| +|charset | |不支持动态修改 |字符集编码，缺省从系统中获取| :::info 1. 为应对多时区的数据写入和查询问题，TDengine 采用 Unix 时间戳(Unix Timestamp)来记录和存储时间戳。Unix 时间戳的特点决定了任一时刻不论在任何时区，产生的时间戳均一致。需要注意的是，Unix 时间戳是在客户端完成转换和记录。为了确保客户端其他形式的时间转换为正确的 Unix 时间戳，需要设置正确的时区。 @@ -162,159 +161,160 @@ charset 的有效值是 UTF-8。 ::: ### 存储相关 -|参数名称|支持版本|参数含义| -|--------------------|----------|-| -|dataDir | |数据文件目录，所有的数据文件都将写入该目录，默认值 /var/lib/taos| -|tempDir | |指定所有系统运行过程中的临时文件生成的目录，默认值 /tmp| -|minimalDataDirGB | |dataDir 指定的时序数据存储目录所需要保留的最小空间，单位 GB，默认值 2| -|minimalTmpDirGB | |tempDir 所指定的临时文件目录所需要保留的最小空间，单位 GB，默认值 1| -|minDiskFreeSize |3.1.1.0 后|当某块磁盘上的可用空间小于等于这个阈值时，该磁盘将不再被选择用于生成新的数据文件，单位为字节，取值范围 52428800-1073741824，默认值为 52428800；企业版参数| -|s3MigrateIntervalSec|3.3.4.3 后|本地数据文件自动上传 S3 的触发周期，单位为秒。最小值：600；最大值：100000。默认值 3600；企业版参数| -|s3MigrateEnabled |3.3.4.3 后|是否自动进行 S3 迁移，默认值为 0，表示关闭自动 S3 迁移，可配置为 1；企业版参数| -|s3Accesskey |3.3.4.3 后|冒号分隔的用户 SecretId:SecretKey，例如 AKIDsQmwsfKxTo2A6nGVXZN0UlofKn6JRRSJ:lIdoy99ygEacU7iHfogaN2Xq0yumSm1E；企业版参数| -|s3Endpoint |3.3.4.3 后|用户所在地域的 COS 服务域名，支持 http 和 https，bucket 的区域需要与 endpoint 保持一致，否则无法访问；企业版参数| -|s3BucketName |3.3.4.3 后|存储桶名称，减号后面是用户注册 COS 服务的 AppId，其中 AppId 是 COS 特有，AWS 和阿里云都没有，配置时需要作为 bucket name 的一部分，使用减号分隔；参数值均为字符串类型，但不需要引号；例如 test0711-1309024725；企业版参数| -|s3PageCacheSize |3.3.4.3 后|S3 page cache 缓存页数目，取值范围 4-1048576，单位为页，默认值 4096；企业版参数| -|s3UploadDelaySec |3.3.4.3 后|data 文件持续多长时间不再变动后上传至 S3，取值范围 1-2592000 (30天），单位为秒，默认值 60；企业版参数| -|cacheLazyLoadThreshold| |内部参数，缓存的装载策略| +|参数名称|支持版本|动态修改|参数含义| +|--------------------|----------|-------------------------|-| +|dataDir | |不支持动态修改 |数据文件目录，所有的数据文件都将写入该目录，默认值 /var/lib/taos| +|tempDir | |不支持动态修改 |指定所有系统运行过程中的临时文件生成的目录，默认值 /tmp| +|minimalDataDirGB | |不支持动态修改 |dataDir 指定的时序数据存储目录所需要保留的最小空间，单位 GB，默认值 2| +|minimalTmpDirGB | |不支持动态修改 |tempDir 所指定的临时文件目录所需要保留的最小空间，单位 GB，默认值 1| +|minDiskFreeSize |3.1.1.0 后|支持动态修改 立即生效 |当某块磁盘上的可用空间小于等于这个阈值时，该磁盘将不再被选择用于生成新的数据文件，单位为字节，取值范围 52428800-1073741824，默认值为 52428800；企业版参数| +|s3MigrateIntervalSec|3.3.4.3 后|支持动态修改 立即生效 |本地数据文件自动上传 S3 的触发周期，单位为秒。最小值：600；最大值：100000。默认值 3600；企业版参数| +|s3MigrateEnabled |3.3.4.3 后|支持动态修改 立即生效 |是否自动进行 S3 迁移，默认值为 0，表示关闭自动 S3 迁移，可配置为 1；企业版参数| +|s3Accesskey |3.3.4.3 后|支持动态修改 重启生效 |冒号分隔的用户 SecretId:SecretKey，例如 AKIDsQmwsfKxTo2A6nGVXZN0UlofKn6JRRSJ:lIdoy99ygEacU7iHfogaN2Xq0yumSm1E；企业版参数| +|s3Endpoint |3.3.4.3 后|支持动态修改 重启生效 |用户所在地域的 COS 服务域名，支持 http 和 https，bucket 的区域需要与 endpoint 保持一致，否则无法访问；企业版参数| +|s3BucketName |3.3.4.3 后|支持动态修改 重启生效 |存储桶名称，减号后面是用户注册 COS 服务的 AppId，其中 AppId 是 COS 特有，AWS 和阿里云都没有，配置时需要作为 bucket name 的一部分，使用减号分隔；参数值均为字符串类型，但不需要引号；例如 test0711-1309024725；企业版参数| +|s3PageCacheSize |3.3.4.3 后|支持动态修改 重启生效 |S3 page cache 缓存页数目，取值范围 4-1048576，单位为页，默认值 4096；企业版参数| +|s3UploadDelaySec |3.3.4.3 后|支持动态修改 立即生效 |data 文件持续多长时间不再变动后上传至 S3，取值范围 1-2592000 (30天），单位为秒，默认值 60；企业版参数| +|cacheLazyLoadThreshold| |支持动态修改 立即生效 |内部参数，缓存的装载策略| ### 集群相关 -|参数名称|支持版本|参数含义| -|--------------------------|----------|-| -|supportVnodes | |dnode 支持的最大 vnode 数目，取值范围 0-4096，默认值 CPU 核数的 2 倍 + 5| -|numOfCommitThreads | |落盘线程的最大数量，取值范围 0-1024，默认值为 4| -|numOfMnodeReadThreads | |mnode 的 Read 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）| -|numOfVnodeQueryThreads | |vnode 的 Query 线程数目，取值范围 0-1024，默认值为 CPU 核数的两倍（不超过 16）| -|numOfVnodeFetchThreads | |vnode 的 Fetch 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）| -|numOfVnodeRsmaThreads | |vnode 的 Rsma 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）| -|numOfQnodeQueryThreads | |qnode 的 Query 线程数目，取值范围 0-1024，默认值为 CPU 核数的两倍（不超过 16）| -|numOfSnodeSharedThreads | |snode 的共享线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不小于 2，不超过 4）| -|numOfSnodeUniqueThreads | |snode 的独占线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不小于 2，不超过 4）| -|ratioOfVnodeStreamThreads | |流计算使用 vnode 线程的比例，取值范围 0.01-4，默认值 4| -|ttlUnit | |ttl 参数的单位，取值范围 1-31572500，单位为秒，默认值 86400| -|ttlPushInterval | |ttl 检测超时频率，取值范围 1-100000，单位为秒，默认值 10| -|ttlChangeOnWrite | |ttl 到期时间是否伴随表的修改操作改变；0：不改变，1：改变；默认值为 0| -|ttlBatchDropNum | |ttl 一批删除子表的数目，最小值为 0，默认值 10000| -|retentionSpeedLimitMB | |数据在不同级别硬盘上迁移时的速度限制，取值范围 0-1024，单位 MB，默认值 0，表示不限制| -|maxTsmaNum | |集群内可创建的TSMA个数；取值范围 0-3；默认值 3| -|tmqMaxTopicNum | |订阅最多可建立的 topic 数量；取值范围 1-10000；默认值为 20| -|tmqRowSize | |订阅数据块的最大记录条数，取值范围 1-1000000，默认值 4096| -|audit | |审计功能开关；企业版参数| -|auditInterval | |审计数据上报的时间间隔；企业版参数| -|auditCreateTable | |是否针对创建子表开启申计功能；企业版参数| -|encryptAlgorithm | |数据加密算法；企业版参数| -|encryptScope | |加密范围；企业版参数| -|enableWhiteList | |白名单功能开关；企业版参数| -|syncLogBufferMemoryAllowed| |一个 dnode 允许的 sync 日志缓存消息占用的内存最大值，单位 bytes，取值范围 104857600-INT64_MAX，默认值 服务器内存的 1/10，3.1.3.2/3.3.2.13 版本开始生效 | -|syncElectInterval | |内部参数，用于同步模块调试| -|syncHeartbeatInterval | |内部参数，用于同步模块调试| -|syncHeartbeatTimeout | |内部参数，用于同步模块调试| -|syncSnapReplMaxWaitN | |内部参数，用于同步模块调试| -|syncSnapReplMaxWaitN | |内部参数，用于同步模块调试| -|arbHeartBeatIntervalSec | |内部参数，用于同步模块调试| -|arbCheckSyncIntervalSec | |内部参数，用于同步模块调试| -|arbSetAssignedTimeoutSec | |内部参数，用于同步模块调试| -|mndSdbWriteDelta | |内部参数，用于 mnode 模块调试| -|mndLogRetention | |内部参数，用于 mnode 模块调试| -|skipGrant | |内部参数，用于授权检查| -|trimVDbIntervalSec | |内部参数，用于删除过期数据| -|ttlFlushThreshold | |内部参数，ttl 定时器的频率| -|compactPullupInterval | |内部参数，数据重整定时器的频率| -|walFsyncDataSizeLimit | |内部参数，WAL 进行 FSYNC 的阈值| -|transPullupInterval | |内部参数，mnode 执行事务的重试间隔| -|mqRebalanceInterval | |内部参数，消费者再平衡的时间间隔| -|uptimeInterval | |内部参数，用于记录系统启动时间| -|timeseriesThreshold | |内部参数，用于统计用量| -|udf | |是否启动 UDF 服务；0：不启动，1：启动；默认值为 0 | -|udfdResFuncs | |内部参数，用于 UDF 结果集设置| -|udfdLdLibPath | |内部参数，表示 UDF 装载的库路径| +|参数名称|支持版本|动态修改|参数含义| +|--------------------------|----------|-------------------------|-| +|supportVnodes | |支持动态修改 立即生效 |dnode 支持的最大 vnode 数目，取值范围 0-4096，默认值 CPU 核数的 2 倍 + 5| +|numOfCommitThreads | |支持动态修改 重启生效 |落盘线程的最大数量，取值范围 0-1024，默认值为 4| +|numOfMnodeReadThreads | |支持动态修改 重启生效 |mnode 的 Read 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）| +|numOfVnodeQueryThreads | |支持动态修改 重启生效 |vnode 的 Query 线程数目，取值范围 0-1024，默认值为 CPU 核数的两倍（不超过 16）| +|numOfVnodeFetchThreads | |支持动态修改 重启生效 |vnode 的 Fetch 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）| +|numOfVnodeRsmaThreads | |支持动态修改 重启生效 |vnode 的 Rsma 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）| +|numOfQnodeQueryThreads | |支持动态修改 重启生效 |qnode 的 Query 线程数目，取值范围 0-1024，默认值为 CPU 核数的两倍（不超过 16）| +|numOfSnodeSharedThreads | |支持动态修改 重启生效 |snode 的共享线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不小于 2，不超过 4）| +|numOfSnodeUniqueThreads | |支持动态修改 重启生效 |snode 的独占线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不小于 2，不超过 4）| +|ratioOfVnodeStreamThreads | |支持动态修改 重启生效 |流计算使用 vnode 线程的比例，取值范围 0.01-4，默认值 4| +|ttlUnit | |不支持动态修改 |ttl 参数的单位，取值范围 1-31572500，单位为秒，默认值 86400| +|ttlPushInterval | |支持动态修改 立即生效 |ttl 检测超时频率，取值范围 1-100000，单位为秒，默认值 10| +|ttlChangeOnWrite | |支持动态修改 立即生效 |ttl 到期时间是否伴随表的修改操作改变；0：不改变，1：改变；默认值为 0| +|ttlBatchDropNum | |支持动态修改 立即生效 |ttl 一批删除子表的数目，最小值为 0，默认值 10000| +|retentionSpeedLimitMB | |支持动态修改 立即生效 |数据在不同级别硬盘上迁移时的速度限制，取值范围 0-1024，单位 MB，默认值 0，表示不限制| +|maxTsmaNum | |支持动态修改 立即生效 |集群内可创建的TSMA个数；取值范围 0-3；默认值 3| +|tmqMaxTopicNum | |支持动态修改 立即生效 |订阅最多可建立的 topic 数量；取值范围 1-10000；默认值为 20| +|tmqRowSize | |支持动态修改 立即生效 |订阅数据块的最大记录条数，取值范围 1-1000000，默认值 4096| +|audit | |支持动态修改 立即生效 |审计功能开关；企业版参数| +|auditInterval | |支持动态修改 立即生效 |审计数据上报的时间间隔；企业版参数| +|auditCreateTable | |支持动态修改 立即生效 |是否针对创建子表开启申计功能；企业版参数| +|encryptAlgorithm | |不支持动态修改 |数据加密算法；企业版参数| +|encryptScope | |不支持动态修改 |加密范围；企业版参数| +|enableWhiteList | |支持动态修改 立即生效 |白名单功能开关；企业版参数| +|syncLogBufferMemoryAllowed| |支持动态修改 立即生效 |一个 dnode 允许的 sync 日志缓存消息占用的内存最大值，单位 bytes，取值范围 104857600-INT64_MAX，默认值 服务器内存的 1/10，3.1.3.2/3.3.2.13 版本开始生效 | +|syncElectInterval | |不支持动态修改 |内部参数，用于同步模块调试| +|syncHeartbeatInterval | |不支持动态修改 |内部参数，用于同步模块调试| +|syncHeartbeatTimeout | |不支持动态修改 |内部参数，用于同步模块调试| +|syncSnapReplMaxWaitN | |支持动态修改 立即生效 |内部参数，用于同步模块调试| +|arbHeartBeatIntervalSec | |支持动态修改 立即生效 |内部参数，用于同步模块调试| +|arbCheckSyncIntervalSec | |支持动态修改 立即生效 |内部参数，用于同步模块调试| +|arbSetAssignedTimeoutSec | |支持动态修改 立即生效 |内部参数，用于同步模块调试| +|mndSdbWriteDelta | |支持动态修改 立即生效 |内部参数，用于 mnode 模块调试| +|mndLogRetention | |支持动态修改 立即生效 |内部参数，用于 mnode 模块调试| +|skipGrant | |不支持动态修改 |内部参数，用于授权检查| +|trimVDbIntervalSec | |支持动态修改 立即生效 |内部参数，用于删除过期数据| +|ttlFlushThreshold | |支持动态修改 立即生效 |内部参数，ttl 定时器的频率| +|compactPullupInterval | |支持动态修改 立即生效 |内部参数，数据重整定时器的频率| +|walFsyncDataSizeLimit | |支持动态修改 立即生效 |内部参数，WAL 进行 FSYNC 的阈值| +|transPullupInterval | |支持动态修改 立即生效 |内部参数，mnode 执行事务的重试间隔| +|mqRebalanceInterval | |支持动态修改 立即生效 |内部参数，消费者再平衡的时间间隔| +|uptimeInterval | |支持动态修改 立即生效 |内部参数，用于记录系统启动时间| +|timeseriesThreshold | |支持动态修改 立即生效 |内部参数，用于统计用量| +|udf | |支持动态修改 重启生效 |是否启动 UDF 服务；0：不启动，1：启动；默认值为 0 | +|udfdResFuncs | |支持动态修改 重启生效 |内部参数，用于 UDF 结果集设置| +|udfdLdLibPath | |支持动态修改 重启生效 |内部参数，表示 UDF 装载的库路径| ### 流计算参数 -|参数名称|支持版本|参数含义| -|-----------------------|----------|-| -|disableStream | |流计算的启动开关| -|streamBufferSize | |控制内存中窗口状态缓存的大小，默认值为 128MB| -|streamAggCnt | |内部参数，并发进行聚合计算的数目| -|checkpointInterval | |内部参数，checkponit 同步间隔| -|concurrentCheckpoint | |内部参数，是否并发检查 checkpoint| -|maxStreamBackendCache | |内部参数，流计算使用的最大缓存| -|streamSinkDataRate | |内部参数，用于控制流计算结果的写入速度| +|参数名称|支持版本|动态修改|参数含义| +|-----------------------|-------------------------|----------|-| +|disableStream | |支持动态修改 重启生效 |流计算的启动开关| +|streamBufferSize | |支持动态修改 重启生效 |控制内存中窗口状态缓存的大小，默认值为 128MB| +|streamAggCnt | |不支持动态修改 |内部参数，并发进行聚合计算的数目| +|checkpointInterval | |支持动态修改 重启生效 |内部参数，checkponit 同步间隔| +|concurrentCheckpoint | |支持动态修改 立即生效 |内部参数，是否并发检查 checkpoint| +|maxStreamBackendCache | |支持动态修改 立即生效 |内部参数，流计算使用的最大缓存| +|streamSinkDataRate | |支持动态修改 重启生效 |内部参数，用于控制流计算结果的写入速度| ### 日志相关 -|参数名称|支持版本|参数含义| -|----------------|----------|-| -|logDir | |日志文件目录，运行日志将写入该目录，默认值 /var/log/taos| -|minimalLogDirGB | |日志文件夹所在磁盘可用空间大小小于该值时，停止写日志，单位 GB，默认值 1| -|numOfLogLines | |单个日志文件允许的最大行数，默认值 10,000,000| -|asyncLog | |日志写入模式，0：同步，1：异步，默认值 1| -|logKeepDays | |日志文件的最长保存时间，单位：天，默认值 0，意味着无限保存，日志文件不会被重命名，也不会有新的日志文件滚动产生，但日志文件的内容有可能会不断滚动，取决于日志文件大小的设置；当设置为大于 0 的值时，当日志文件大小达到设置的上限时会被重命名为 taosdlog.yyy，其中 yyy 为日志文件最后修改的时间戳，并滚动产生新的日志文件| -|slowLogThreshold|3.3.3.0 后|慢查询门限值，大于等于门限值认为是慢查询，单位秒，默认值 3 | -|slowLogMaxLen |3.3.3.0 后|慢查询日志最大长度，取值范围 1-16384，默认值 4096| -|slowLogScope |3.3.3.0 后|慢查询记录类型，取值范围 ALL/QUERY/INSERT/OTHERS/NONE，默认值 QUERY| -|slowLogExceptDb |3.3.3.0 后|指定的数据库不上报慢查询，仅支持配置换一个数据库| -|debugFlag | |运行日志开关，131（输出错误和警告日志），135（输出错误、警告和调试日志），143（输出错误、警告、调试和跟踪日志）；默认值 131 或 135 （取决于不同模块）| -|tmrDebugFlag | |定时器模块的日志开关，取值范围同上| -|uDebugFlag | |共用功能模块的日志开关，取值范围同上| -|rpcDebugFlag | |rpc 模块的日志开关，取值范围同上| -|qDebugFlag | |query 模块的日志开关，取值范围同上| -|dDebugFlag | |dnode 模块的日志开关，取值范围同上| -|vDebugFlag | |vnode 模块的日志开关，取值范围同上| -|mDebugFlag | |mnode 模块的日志开关，取值范围同上| -|azDebugFlag |3.3.4.3 后|S3 模块的日志开关，取值范围同上| -|sDebugFlag | |sync 模块的日志开关，取值范围同上| -|tsdbDebugFlag | |tsdb 模块的日志开关，取值范围同上| -|tqDebugFlag | |tq 模块的日志开关，取值范围同上| -|fsDebugFlag | |fs 模块的日志开关，取值范围同上| -|udfDebugFlag | |udf 模块的日志开关，取值范围同上| -|smaDebugFlag | |sma 模块的日志开关，取值范围同上| -|idxDebugFlag | |index 模块的日志开关，取值范围同上| -|tdbDebugFlag | |tdb 模块的日志开关，取值范围同上| -|metaDebugFlag | |meta 模块的日志开关，取值范围同上| -|stDebugFlag | |stream 模块的日志开关，取值范围同上| -|sndDebugFlag | |snode 模块的日志开关，取值范围同上| +|参数名称|支持版本|动态修改|参数含义| +|----------------|-------------------------|----------|-| +|logDir | |不支持动态修改 |日志文件目录，运行日志将写入该目录，默认值 /var/log/taos| +|minimalLogDirGB | ｜不支持动态修改 |日志文件夹所在磁盘可用空间大小小于该值时，停止写日志，单位 GB，默认值 1| +|numOfLogLines | ｜支持动态修改 立即生效 |单个日志文件允许的最大行数，默认值 10,000,000| +|asyncLog | |支持动态修改 立即生效 |日志写入模式，0：同步，1：异步，默认值 1| +|logKeepDays | ｜支持动态修改 立即生效 |日志文件的最长保存时间，单位：天，默认值 0，意味着无限保存，日志文件不会被重命名，也不会有新的日志文件滚动产生，但日志文件的内容有可能会不断滚动，取决于日志文件大小的设置；当设置为大于 0 的值时，当日志文件大小达到设置的上限时会被重命名为 taosdlog.yyy，其中 yyy 为日志文件最后修改的时间戳，并滚动产生新的日志文件| +|slowLogThreshold|3.3.3.0 后|支持动态修改 立即生效 |慢查询门限值，大于等于门限值认为是慢查询，单位秒，默认值 3 | +|slowLogMaxLen |3.3.3.0 后|支持动态修改 立即生效 |慢查询日志最大长度，取值范围 1-16384，默认值 4096| +|slowLogScope |3.3.3.0 后|支持动态修改 立即生效 |慢查询记录类型，取值范围 ALL/QUERY/INSERT/OTHERS/NONE，默认值 QUERY| +|slowLogExceptDb |3.3.3.0 后|支持动态修改 立即生效 |指定的数据库不上报慢查询，仅支持配置换一个数据库| +|debugFlag | |支持动态修改 立即生效 |运行日志开关，131（输出错误和警告日志），135（输出错误、警告和调试日志），143（输出错误、警告、调试和跟踪日志）；默认值 131 或 135 （取决于不同模块）| +|tmrDebugFlag | |支持动态修改 立即生效 |定时器模块的日志开关，取值范围同上| +|uDebugFlag | |支持动态修改 立即生效 |共用功能模块的日志开关，取值范围同上| +|rpcDebugFlag | |支持动态修改 立即生效 |rpc 模块的日志开关，取值范围同上| +|qDebugFlag | |支持动态修改 立即生效 |query 模块的日志开关，取值范围同上| +|dDebugFlag | |支持动态修改 立即生效 |dnode 模块的日志开关，取值范围同上| +|vDebugFlag | |支持动态修改 立即生效 |vnode 模块的日志开关，取值范围同上| +|mDebugFlag | |支持动态修改 立即生效 |mnode 模块的日志开关，取值范围同上| +|azDebugFlag |3.3.4.3 后|支持动态修改 立即生效 |S3 模块的日志开关，取值范围同上| +|sDebugFlag | |支持动态修改 立即生效 |sync 模块的日志开关，取值范围同上| +|tsdbDebugFlag | |支持动态修改 立即生效 |tsdb 模块的日志开关，取值范围同上| +|tqDebugFlag | |支持动态修改 立即生效 |tq 模块的日志开关，取值范围同上| +|fsDebugFlag | |支持动态修改 立即生效 |fs 模块的日志开关，取值范围同上| +|udfDebugFlag | |支持动态修改 立即生效 |udf 模块的日志开关，取值范围同上| +|smaDebugFlag | |支持动态修改 立即生效 |sma 模块的日志开关，取值范围同上| +|idxDebugFlag | |支持动态修改 立即生效 |index 模块的日志开关，取值范围同上| +|tdbDebugFlag | |支持动态修改 立即生效 |tdb 模块的日志开关，取值范围同上| +|metaDebugFlag | |支持动态修改 立即生效 |meta 模块的日志开关，取值范围同上| +|stDebugFlag | |支持动态修改 立即生效 |stream 模块的日志开关，取值范围同上| +|sndDebugFlag | |支持动态修改 立即生效 |snode 模块的日志开关，取值范围同上| ### 调试相关 -|参数名称|支持版本|参数含义| -|--------------------|----------|-| -|enableCoreFile | |crash 时是否生成 core 文件，0：不生成，1：生成；默认值 1| -|configDir | |配置文件所在目录| -|scriptDir | |内部测试工具的脚本目录| -|assert | |断言控制开关，默认值 0| -|randErrorChance | |内部参数，用于随机失败测试| -|randErrorDivisor | |内部参数，用于随机失败测试| -|randErrorScope | |内部参数，用于随机失败测试| -|safetyCheckLevel | |内部参数，用于随机失败测试| -|experimental | |内部参数，用于一些实验特性| -|simdEnable |3.3.4.3 后|内部参数，用于测试 SIMD 加速| -|AVX512Enable |3.3.4.3 后|内部参数，用于测试 AVX512 加速| -|rsyncPort | |内部参数，用于调试流计算| -|snodeAddress | |内部参数，用于调试流计算| -|checkpointBackupDir | |内部参数，用于恢复 snode 数据| -|enableAuditDelete | |内部参数，用于测试审计功能| -|slowLogThresholdTest| |内部参数，用于测试慢日志| -|bypassFlag |3.3.4.5 后|内部参数，用于短路测试，0：正常写入，1：写入消息在 taos 客户端发送 RPC 消息前返回，2：写入消息在 taosd 服务端收到 RPC 消息后返回，4：写入消息在 taosd 服务端写入内存缓存前返回，8：写入消息在 taosd 服务端数据落盘前返回；默认值 0| +|参数名称|支持版本|动态修改|参数含义| +|--------------------|-------------------------|----------|-| +|enableCoreFile | |支持动态修改 立即生效 |crash 时是否生成 core 文件，0：不生成，1：生成；默认值 1| +|configDir | |不支持动态修改 |配置文件所在目录| +|forceReadConfig | |不支持动态修改 |强制使用配置文件中的参数，0:使用持久化的配置参数，1:使用配置文件中的配置参数；默认值 0| +|scriptDir | |不支持动态修改 |内部测试工具的脚本目录| +|assert | |不支持动态修改 |断言控制开关，默认值 0| +|randErrorChance | |支持动态修改 立即生效 |内部参数，用于随机失败测试| +|randErrorDivisor | |支持动态修改 立即生效 |内部参数，用于随机失败测试| +|randErrorScope | |支持动态修改 立即生效 |内部参数，用于随机失败测试| +|safetyCheckLevel | |支持动态修改 立即生效 |内部参数，用于随机失败测试| +|experimental | |支持动态修改 立即生效 |内部参数，用于一些实验特性| +|simdEnable |3.3.4.3 后|不支持动态修改 |内部参数，用于测试 SIMD 加速| +|AVX512Enable |3.3.4.3 后|不支持动态修改 |内部参数，用于测试 AVX512 加速| +|rsyncPort | |不支持动态修改 |内部参数，用于调试流计算| +|snodeAddress | |支持动态修改 重启生效 |内部参数，用于调试流计算| +|checkpointBackupDir | |支持动态修改 重启生效 |内部参数，用于恢复 snode 数据| +|enableAuditDelete | |不支持动态修改 |内部参数，用于测试审计功能| +|slowLogThresholdTest| |不支持动态修改 |内部参数，用于测试慢日志| +|bypassFlag |3.3.4.5 后|支持动态修改 立即生效 |内部参数，用于短路测试，0：正常写入，1：写入消息在 taos 客户端发送 RPC 消息前返回，2：写入消息在 taosd 服务端收到 RPC 消息后返回，4：写入消息在 taosd 服务端写入内存缓存前返回，8：写入消息在 taosd 服务端数据落盘前返回；默认值 0| ### 压缩参数 -|参数名称|支持版本|参数含义| -|------------|----------|-| -|fPrecision | |设置 float 类型浮点数压缩精度 ，取值范围 0.1 ~ 0.00000001 ，默认值 0.00000001 , 小于此值的浮点数尾数部分将被截断| -|dPrecision | |设置 double 类型浮点数压缩精度 , 取值范围 0.1 ~ 0.0000000000000001 ， 默认值 0.0000000000000001 ， 小于此值的浮点数尾数部分将被截取| -|lossyColumn |3.3.0.0 前|对 float 和/或 double 类型启用 TSZ 有损压缩；取值范围 float/double/none；默认值 none，表示关闭无损压缩| -|ifAdtFse | |在启用 TSZ 有损压缩时，使用 FSE 算法替换 HUFFMAN 算法，FSE 算法压缩速度更快，但解压稍慢，追求压缩速度可选用此算法；0：关闭，1：打开；默认值为 0| -|maxRange | |内部参数，用于有损压缩设置| -|curRange | |内部参数，用于有损压缩设置| -|compressor | |内部参数，用于有损压缩设置| +|参数名称|支持版本|动态修改|参数含义| +|------------|----------|-------------------------|-| +|fPrecision | |支持动态修改 立即生效 |设置 float 类型浮点数压缩精度 ，取值范围 0.1 ~ 0.00000001 ，默认值 0.00000001 , 小于此值的浮点数尾数部分将被截断| +|dPrecision | |支持动态修改 立即生效 |设置 double 类型浮点数压缩精度 , 取值范围 0.1 ~ 0.0000000000000001 ， 默认值 0.0000000000000001 ， 小于此值的浮点数尾数部分将被截取| +|lossyColumn |3.3.0.0 前|不支持动态修改 |对 float 和/或 double 类型启用 TSZ 有损压缩；取值范围 float/double/none；默认值 none，表示关闭无损压缩| +|ifAdtFse | |支持动态修改 重启生效 |在启用 TSZ 有损压缩时，使用 FSE 算法替换 HUFFMAN 算法，FSE 算法压缩速度更快，但解压稍慢，追求压缩速度可选用此算法；0：关闭，1：打开；默认值为 0| +|maxRange | |支持动态修改 重启生效 |内部参数，用于有损压缩设置| +|curRange | |支持动态修改 重启生效 |内部参数，用于有损压缩设置| +|compressor | |支持动态修改 重启生效 |内部参数，用于有损压缩设置| **补充说明** -1. 在 3.2.0.0 ~ 3.3.0.0（不包含）版本生效，启用该参数后不能回退到升级前的版本 -2. TSZ 压缩算法是通过数据预测技术完成的压缩，所以更适合有规律变化的数据 -3. TSZ 压缩时间会更长一些，如果您的服务器 CPU 空闲多，存储空间小的情况下适合选用 -4. 示例：对 float 和 double 类型都启用有损压缩 +1. 在 3.4.0.0 之后，所有配置参数都将被持久化到本地存储，重启数据库服务后，将默认使用持久化的配置参数列表；如果您希望继续使用 config 文件中配置的参数，需设置 forceReadConfig 为 1。 +2. 在 3.2.0.0 ~ 3.3.0.0（不包含）版本生效，启用该参数后不能回退到升级前的版本 +3. TSZ 压缩算法是通过数据预测技术完成的压缩，所以更适合有规律变化的数据 +4. TSZ 压缩时间会更长一些，如果您的服务器 CPU 空闲多，存储空间小的情况下适合选用 +5. 示例：对 float 和 double 类型都启用有损压缩 ```shell lossyColumns float|double ``` -5. 配置需重启服务生效，重启如果在 taosd 日志中看到以下内容，表明配置已生效： +6. 配置需重启服务生效，重启如果在 taosd 日志中看到以下内容，表明配置已生效： ```sql 02/22 10:49:27.607990 00002933 UTL lossyColumns float|double ``` diff --git a/docs/zh/14-reference/01-components/02-taosc.md b/docs/zh/14-reference/01-components/02-taosc.md index 6b6f621aaf..46b8854d27 100755 --- a/docs/zh/14-reference/01-components/02-taosc.md +++ b/docs/zh/14-reference/01-components/02-taosc.md @@ -9,101 +9,102 @@ TDengine 客户端驱动提供了应用编程所需要的全部 API，并且在 ## 配置参数 ### 连接相关 -|参数名称|支持版本|参数含义| -|----------------------|----------|-------------| -|firstEp | |启动时，主动连接的集群中首个 dnode 的 endpoint，缺省值：hostname:6030，若无法获取该服务器的 hostname，则赋值为 localhost| -|secondEp | |启动时，如果 firstEp 连接不上，尝试连接集群中第二个 dnode 的 endpoint，没有缺省值| -|compressMsgSize | |是否对 RPC 消息进行压缩；-1：所有消息都不压缩；0：所有消息都压缩；N (N>0)：只有大于 N 个字节的消息才压缩；缺省值 -1| -|shellActivityTimer | |客户端向 mnode 发送心跳的时长，单位为秒，取值范围 1-120，默认值 3| -|numOfRpcSessions | |RPC 支持的最大连接数，取值范围 100-100000，缺省值 30000| -|numOfRpcThreads | |RPC 收发数据线程数目，取值范围1-50,默认值为 CPU 核数的一半| -|numOfTaskQueueThreads | |客户端处理 RPC消息的线程数, 范围4-16,默认值为 CPU 核数的一半| -|timeToGetAvailableConn| 3.3.4.*之后取消 |获得可用连接的最长等待时间，取值范围 10-50000000，单位为毫秒，缺省值 500000| -|useAdapter | |内部参数，是否使用 taosadapter，影响 CSV 文件导入| -|shareConnLimit |3.3.4.0 新增|内部参数，一个链接可以共享的查询数目，取值范围 1-256，默认值 10| -|readTimeout |3.3.4.0 新增|内部参数，最小超时时间，取值范围 64-604800，单位为秒，默认值 900| +|参数名称|支持版本|动态修改|参数含义| +|----------------------|----------|-------------------------|-------------| +|firstEp | |支持动态修改 立即生效 |启动时，主动连接的集群中首个 dnode 的 endpoint，缺省值：hostname:6030，若无法获取该服务器的 hostname，则赋值为 localhost| +|secondEp | |支持动态修改 立即生效 |启动时，如果 firstEp 连接不上，尝试连接集群中第二个 dnode 的 endpoint，没有缺省值| +|serverPort | |支持动态修改 立即生效 |taosd 监听的端口，默认值 6030| +|compressMsgSize | |支持动态修改 立即生效 |是否对 RPC 消息进行压缩；-1：所有消息都不压缩；0：所有消息都压缩；N (N>0)：只有大于 N 个字节的消息才压缩；缺省值 -1| +|shellActivityTimer | |不支持动态修改 |客户端向 mnode 发送心跳的时长，单位为秒，取值范围 1-120，默认值 3| +|numOfRpcSessions | |支持动态修改 立即生效 |RPC 支持的最大连接数，取值范围 100-100000，缺省值 30000| +|numOfRpcThreads | |不支持动态修改 |RPC 收发数据线程数目，取值范围1-50,默认值为 CPU 核数的一半| +|numOfTaskQueueThreads | |不支持动态修改 |客户端处理 RPC消息的线程数, 范围4-16,默认值为 CPU 核数的一半| +|timeToGetAvailableConn| 3.3.4.*之后取消 |不支持动态修改 |获得可用连接的最长等待时间，取值范围 10-50000000，单位为毫秒，缺省值 500000| +|useAdapter | |支持动态修改 立即生效 |内部参数，是否使用 taosadapter，影响 CSV 文件导入| +|shareConnLimit |3.3.4.0 新增|不支持动态修改 |内部参数，一个链接可以共享的查询数目，取值范围 1-256，默认值 10| +|readTimeout |3.3.4.0 新增|不支持动态修改 |内部参数，最小超时时间，取值范围 64-604800，单位为秒，默认值 900| ### 查询相关 -|参数名称|支持版本|参数含义| -|---------------------------------|---------|-| -|countAlwaysReturnValue | |count/hyperloglog 函数在输入数据为空或者 NULL 的情况下是否返回值；0：返回空行，1：返回；默认值 1；该参数设置为 1 时，如果查询中含有 INTERVAL 子句或者该查询使用了 TSMA 时，且相应的组或窗口内数据为空或者 NULL，对应的组或窗口将不返回查询结果；注意此参数客户端和服务端值应保持一致| -|keepColumnName | |Last、First、LastRow 函数查询且未指定别名时，自动设置别名为列名（不含函数名），因此 order by 子句如果引用了该列名将自动引用该列对应的函数；1：表示自动设置别名为列名(不包含函数名)，0：表示不自动设置别名；缺省值：0| -|multiResultFunctionStarReturnTags|3.3.3.0 后|查询超级表时，last(\*)/last_row(\*)/first(\*) 是否返回标签列；查询普通表、子表时，不受该参数影响；0：不返回标签列，1：返回标签列；缺省值：0；该参数设置为 0 时，last(\*)/last_row(\*)/first(\*) 只返回超级表的普通列；为 1 时，返回超级表的普通列和标签列| -|metaCacheMaxSize | |指定单个客户端元数据缓存大小的最大值，单位 MB；缺省值 -1，表示无限制| -|maxTsmaCalcDelay | |查询时客户端可允许的 tsma 计算延迟，若 tsma 的计算延迟大于配置值，则该 TSMA 将不会被使用；取值范围 600s - 86400s，即 10 分钟 - 1 小时；缺省值：600 秒| -|tsmaDataDeleteMark | |TSMA 计算的历史数据中间结果保存时间，单位为毫秒；取值范围 >= 3600000，即大于等于1h；缺省值：86400000，即 1d | -|queryPolicy | |查询语句的执行策略，1：只使用 vnode，不使用 qnode；2：没有扫描算子的子任务在 qnode 执行，带扫描算子的子任务在 vnode 执行；3：vnode 只运行扫描算子，其余算子均在 qnode 执行；缺省值：1| -|queryTableNotExistAsEmpty | |查询表不存在时是否返回空结果集；false：返回错误；true：返回空结果集；缺省值 false| -|querySmaOptimize | |sma index 的优化策略，0：表示不使用 sma index，永远从原始数据进行查询；1：表示使用 sma index，对符合的语句，直接从预计算的结果进行查询；缺省值：0| -|queryPlannerTrace | |内部参数，查询计划是否输出详细日志| -|queryNodeChunkSize | |内部参数，查询计划的块大小| -|queryUseNodeAllocator | |内部参数，查询计划的分配方法| -|queryMaxConcurrentTables | |内部参数，查询计划的并发数目| -|enableQueryHb | |内部参数，是否发送查询心跳消息| -|minSlidingTime | |内部参数，sliding 的最小允许值| -|minIntervalTime | |内部参数，interval 的最小允许值| +|参数名称|支持版本|动态修改|参数含义| +|----------------------|----------|-------------------------|-------------| +|countAlwaysReturnValue | |支持动态修改 立即生效 |count/hyperloglog 函数在输入数据为空或者 NULL 的情况下是否返回值；0：返回空行，1：返回；默认值 1；该参数设置为 1 时，如果查询中含有 INTERVAL 子句或者该查询使用了 TSMA 时，且相应的组或窗口内数据为空或者 NULL，对应的组或窗口将不返回查询结果；注意此参数客户端和服务端值应保持一致| +|keepColumnName | |支持动态修改 立即生效 |Last、First、LastRow 函数查询且未指定别名时，自动设置别名为列名（不含函数名），因此 order by 子句如果引用了该列名将自动引用该列对应的函数；1：表示自动设置别名为列名(不包含函数名)，0：表示不自动设置别名；缺省值：0| +|multiResultFunctionStarReturnTags|3.3.3.0 后|支持动态修改 立即生效 |查询超级表时，last(\*)/last_row(\*)/first(\*) 是否返回标签列；查询普通表、子表时，不受该参数影响；0：不返回标签列，1：返回标签列；缺省值：0；该参数设置为 0 时，last(\*)/last_row(\*)/first(\*) 只返回超级表的普通列；为 1 时，返回超级表的普通列和标签列| +|metaCacheMaxSize | |支持动态修改 立即生效 |指定单个客户端元数据缓存大小的最大值，单位 MB；缺省值 -1，表示无限制| +|maxTsmaCalcDelay | |支持动态修改 立即生效 |查询时客户端可允许的 tsma 计算延迟，若 tsma 的计算延迟大于配置值，则该 TSMA 将不会被使用；取值范围 600s - 86400s，即 10 分钟 - 1 小时；缺省值：600 秒| +|tsmaDataDeleteMark | |支持动态修改 立即生效 |TSMA 计算的历史数据中间结果保存时间，单位为毫秒；取值范围 >= 3600000，即大于等于1h；缺省值：86400000，即 1d | +|queryPolicy | |支持动态修改 立即生效 |查询语句的执行策略，1：只使用 vnode，不使用 qnode；2：没有扫描算子的子任务在 qnode 执行，带扫描算子的子任务在 vnode 执行；3：vnode 只运行扫描算子，其余算子均在 qnode 执行；缺省值：1| +|queryTableNotExistAsEmpty | |支持动态修改 立即生效 |查询表不存在时是否返回空结果集；false：返回错误；true：返回空结果集；缺省值 false| +|querySmaOptimize | |支持动态修改 立即生效 |sma index 的优化策略，0：表示不使用 sma index，永远从原始数据进行查询；1：表示使用 sma index，对符合的语句，直接从预计算的结果进行查询；缺省值：0| +|queryPlannerTrace | |支持动态修改 立即生效 |内部参数，查询计划是否输出详细日志| +|queryNodeChunkSize | |支持动态修改 立即生效 |内部参数，查询计划的块大小| +|queryUseNodeAllocator | |支持动态修改 立即生效 |内部参数，查询计划的分配方法| +|queryMaxConcurrentTables | |不支持动态修改 |内部参数，查询计划的并发数目| +|enableQueryHb | |支持动态修改 立即生效 |内部参数，是否发送查询心跳消息| +|minSlidingTime | |支持动态修改 立即生效 |内部参数，sliding 的最小允许值| +|minIntervalTime | |支持动态修改 立即生效 |内部参数，interval 的最小允许值| ### 写入相关 -|参数名称|支持版本|参数含义| -|------------------------------|----------|-| -|smlChildTableName | |schemaless 自定义的子表名的 key，无缺省值| -|smlAutoChildTableNameDelimiter| |schemaless tag 之间的连接符，连起来作为子表名，无缺省值| -|smlTagName | |schemaless tag 为空时默认的 tag 名字，缺省值 "_tag_null"| -|smlTsDefaultName | |schemaless 自动建表的时间列名字通过该配置设置，缺省值 "_ts"| -|smlDot2Underline | |schemaless 把超级表名中的 dot 转成下划线| -|maxInsertBatchRows | |内部参数，一批写入的最大条数| +|参数名称|支持版本|动态修改|参数含义| +|----------------------|----------|-------------------------|-------------| +|smlChildTableName | |支持动态修改 立即生效 |schemaless 自定义的子表名的 key，无缺省值| +|smlAutoChildTableNameDelimiter| |支持动态修改 立即生效 |schemaless tag 之间的连接符，连起来作为子表名，无缺省值| +|smlTagName | |支持动态修改 立即生效 |schemaless tag 为空时默认的 tag 名字，缺省值 "_tag_null"| +|smlTsDefaultName | |支持动态修改 立即生效 |schemaless 自动建表的时间列名字通过该配置设置，缺省值 "_ts"| +|smlDot2Underline | |支持动态修改 立即生效 |schemaless 把超级表名中的 dot 转成下划线| +|maxInsertBatchRows | |支持动态修改 立即生效 |内部参数，一批写入的最大条数| ### 区域相关 -|参数名称|支持版本|参数含义| -|-----------------|----------|-| -|timezone | |时区；缺省从系统中动态获取当前的时区设置| -|locale | |系统区位信息及编码格式，缺省从系统中获取| -|charset | |字符集编码，缺省从系统中获取| +|参数名称|支持版本|动态修改|参数含义| +|----------------------|----------|-------------------------|-------------| +|timezone | |支持动态修改 立即生效 |时区；缺省从系统中动态获取当前的时区设置| +|locale | |支持动态修改 立即生效 |系统区位信息及编码格式，缺省从系统中获取| +|charset | |支持动态修改 立即生效 |字符集编码，缺省从系统中获取| ### 存储相关 -|参数名称|支持版本|参数含义| -|-----------------|----------|-| -|tempDir | |指定所有运行过程中的临时文件生成的目录，Linux 平台默认值为 /tmp| -|minimalTmpDirGB | |tempDir 所指定的临时文件目录所需要保留的最小空间，单位 GB，缺省值：1| +|参数名称|支持版本|动态修改|参数含义| +|----------------------|----------|-------------------------|-------------| +|tempDir | |支持动态修改 立即生效 |指定所有运行过程中的临时文件生成的目录，Linux 平台默认值为 /tmp| +|minimalTmpDirGB | |支持动态修改 立即生效 |tempDir 所指定的临时文件目录所需要保留的最小空间，单位 GB，缺省值：1| ### 日志相关 -|参数名称|支持版本|参数含义| -|-----------------|----------|-| -|logDir | |日志文件目录，运行日志将写入该目录，缺省值：/var/log/taos| -|minimalLogDirGB | |日志文件夹所在磁盘可用空间大小小于该值时，停止写日志，单位 GB，缺省值：1| -|numOfLogLines | |单个日志文件允许的最大行数，缺省值：10,000,000| -|asyncLog | |日志写入模式，0：同步，1：异步，缺省值：1| -|logKeepDays | |日志文件的最长保存时间，单位：天，缺省值：0，意味着无限保存，日志文件不会被重命名，也不会有新的日志文件滚动产生，但日志文件的内容有可能会不断滚动，取决于日志文件大小的设置；当设置为大于 0 的值时，当日志文件大小达到设置的上限时会被重命名为 taoslogx.yyy，其中 yyy 为日志文件最后修改的时间戳，并滚动产生新的日志文件| -|debugFlag | |运行日志开关，131（输出错误和警告日志），135（输出错误、警告和调试日志），143（输出错误、警告、调试和跟踪日志）；默认值 131 或 135 （取决于不同模块）| -|tmrDebugFlag | |定时器模块的日志开关，取值范围同上| -|uDebugFlag | |共用功能模块的日志开关，取值范围同上| -|rpcDebugFlag | |rpc 模块的日志开关，取值范围同上| -|jniDebugFlag | |jni 模块的日志开关，取值范围同上| -|qDebugFlag | |query 模块的日志开关，取值范围同上| -|cDebugFlag | |客户端模块的日志开关，取值范围同上| -|simDebugFlag | |内部参数，测试工具的日志开关，取值范围同上| -|tqClientDebugFlag|3.3.4.3 后|客户端模块的日志开关，取值范围同上| +|参数名称|支持版本|动态修改|参数含义| +|----------------------|----------|-------------------------|-------------| +|logDir | |不支持动态修改 |日志文件目录，运行日志将写入该目录，缺省值：/var/log/taos| +|minimalLogDirGB | |支持动态修改 立即生效 |日志文件夹所在磁盘可用空间大小小于该值时，停止写日志，单位 GB，缺省值：1| +|numOfLogLines | |支持动态修改 立即生效 |单个日志文件允许的最大行数，缺省值：10,000,000| +|asyncLog | |支持动态修改 立即生效 |日志写入模式，0：同步，1：异步，缺省值：1| +|logKeepDays | |支持动态修改 立即生效 |日志文件的最长保存时间，单位：天，缺省值：0，意味着无限保存，日志文件不会被重命名，也不会有新的日志文件滚动产生，但日志文件的内容有可能会不断滚动，取决于日志文件大小的设置；当设置为大于 0 的值时，当日志文件大小达到设置的上限时会被重命名为 taoslogx.yyy，其中 yyy 为日志文件最后修改的时间戳，并滚动产生新的日志文件| +|debugFlag | |支持动态修改 立即生效 |运行日志开关，131（输出错误和警告日志），135（输出错误、警告和调试日志），143（输出错误、警告、调试和跟踪日志）；默认值 131 或 135 （取决于不同模块）| +|tmrDebugFlag | |支持动态修改 立即生效 |定时器模块的日志开关，取值范围同上| +|uDebugFlag | |支持动态修改 立即生效 |共用功能模块的日志开关，取值范围同上| +|rpcDebugFlag | |支持动态修改 立即生效 |rpc 模块的日志开关，取值范围同上| +|jniDebugFlag | |支持动态修改 立即生效 |jni 模块的日志开关，取值范围同上| +|qDebugFlag | |支持动态修改 立即生效 |query 模块的日志开关，取值范围同上| +|cDebugFlag | |支持动态修改 立即生效 |客户端模块的日志开关，取值范围同上| +|simDebugFlag | |支持动态修改 立即生效 |内部参数，测试工具的日志开关，取值范围同上| +|tqClientDebugFlag|3.3.4.3 后|支持动态修改 立即生效 |客户端模块的日志开关，取值范围同上| ### 调试相关 -|参数名称|支持版本|参数含义| -|-----------------|-----------|-| -|crashReporting | |是否上传 crash 到 telemetry，0：不上传，1：上传；缺省值：1| -|enableCoreFile | |crash 时是否生成 core 文件，0：不生成，1：生成；缺省值：1| -|assert | |断言控制开关，缺省值：0| -|configDir | |配置文件所在目录| -|scriptDir | |内部参数，测试用例的目录| -|randErrorChance |3.3.3.0 后|内部参数，用于随机失败测试| -|randErrorDivisor |3.3.3.0 后|内部参数，用于随机失败测试| -|randErrorScope |3.3.3.0 后|内部参数，用于随机失败测试| -|safetyCheckLevel |3.3.3.0 后|内部参数，用于随机失败测试| -|simdEnable |3.3.4.3 后|内部参数，用于测试 SIMD 加速| -|AVX512Enable |3.3.4.3 后|内部参数，用于测试 AVX512 加速| -|bypassFlag |3.3.4.5 后|内部参数，用于短路测试，0：正常写入，1：写入消息在 taos 客户端发送 RPC 消息前返回，2：写入消息在 taosd 服务端收到 RPC 消息后返回，4：写入消息在 taosd 服务端写入内存缓存前返回，8：写入消息在 taosd 服务端数据落盘前返回；缺省值：0| +|参数名称|支持版本|动态修改|参数含义| +|----------------------|----------|-------------------------|-------------| +|crashReporting | |支持动态修改 立即生效 |是否上传 crash 到 telemetry，0：不上传，1：上传；缺省值：1| +|enableCoreFile | |支持动态修改 立即生效 |crash 时是否生成 core 文件，0：不生成，1：生成；缺省值：1| +|assert | |不支持动态修改 |断言控制开关，缺省值：0| +|configDir | |不支持动态修改 |配置文件所在目录| +|scriptDir | |不支持动态修改 |内部参数，测试用例的目录| +|randErrorChance |3.3.3.0 后|不支持动态修改 |内部参数，用于随机失败测试| +|randErrorDivisor |3.3.3.0 后|不支持动态修改 |内部参数，用于随机失败测试| +|randErrorScope |3.3.3.0 后|不支持动态修改 |内部参数，用于随机失败测试| +|safetyCheckLevel |3.3.3.0 后|不支持动态修改 |内部参数，用于随机失败测试| +|simdEnable |3.3.4.3 后|不支持动态修改 |内部参数，用于测试 SIMD 加速| +|AVX512Enable |3.3.4.3 后|不支持动态修改 |内部参数，用于测试 AVX512 加速| +|bypassFlag |3.3.4.5 后|支持动态修改 立即生效 |内部参数，用于短路测试，0：正常写入，1：写入消息在 taos 客户端发送 RPC 消息前返回，2：写入消息在 taosd 服务端收到 RPC 消息后返回，4：写入消息在 taosd 服务端写入内存缓存前返回，8：写入消息在 taosd 服务端数据落盘前返回；缺省值：0| ### SHELL 相关 -|参数名称|支持版本|参数含义| -|-----------------|----------|-| -|enableScience | |是否开启科学计数法显示浮点数；0：不开始，1：开启；缺省值：1| +|参数名称|支持版本|动态修改|参数含义| +|----------------------|----------|-------------------------|-------------| +|enableScience | |不支持动态修改 |是否开启科学计数法显示浮点数；0：不开始，1：开启；缺省值：1| ## API diff --git a/docs/zh/14-reference/03-taos-sql/21-node.md b/docs/zh/14-reference/03-taos-sql/21-node.md index e3a672790c..43d805d90e 100644 --- a/docs/zh/14-reference/03-taos-sql/21-node.md +++ b/docs/zh/14-reference/03-taos-sql/21-node.md @@ -42,32 +42,11 @@ DROP DNODE dnode_id [force] [unsafe] ALTER DNODE dnode_id dnode_option ALTER ALL DNODES dnode_option - -dnode_option: { - 'resetLog' - | 'balance' 'value' - | 'monitor' 'value' - | 'debugFlag' 'value' - | 'monDebugFlag' 'value' - | 'vDebugFlag' 'value' - | 'mDebugFlag' 'value' - | 'cDebugFlag' 'value' - | 'httpDebugFlag' 'value' - | 'qDebugflag' 'value' - | 'sdbDebugFlag' 'value' - | 'uDebugFlag' 'value' - | 'tsdbDebugFlag' 'value' - | 'sDebugflag' 'value' - | 'rpcDebugFlag' 'value' - | 'dDebugFlag' 'value' - | 'mqttDebugFlag' 'value' - | 'wDebugFlag' 'value' - | 'tmrDebugFlag' 'value' - | 'cqDebugFlag' 'value' -} ``` -上面语法中的这些可修改配置项其配置方式与 dnode 配置文件中的配置方式相同，区别是修改是动态的立即生效，且不需要重启 dnode。 +对于支持动态修改的配置参数，您可以使用 ALTER DNODE 或 ALTER ALL DNODES 语法修改 dnode 中配置参数的值，自 3.3.4.0 后，修改的配置参数将自动持久化，即便数据库服务重启后仍然生效。 + +对于一个配置参数是否支持动态修改，请您参考以下页面：[taosd 参考手册](../01-components/01-taosd.md) value 是参数的值，需要是字符格式。如修改 dnode 1 的日志输出级别为 debug： @@ -75,6 +54,18 @@ value 是参数的值，需要是字符格式。如修改 dnode 1 的日志输 ALTER DNODE 1 'debugFlag' '143'; ``` +### 补充说明： +配置参数在 dnode 中被分为全局配置参数与局部配置参数，您可以查看 SHOW VARIABLES 或 SHOW DNODE dnode_id VARIABLE 中的 category 字段来确认配置参数属于全局配置参数还是局部配置参数： +1. 局部配置参数：您可以使用 ALTER DNODE 或 ALTER ALL DNODES 来更新某一个 dnode 或全部 dnodes 的局部配置参数。 +2. 全局配置参数：全局配置参数要求各个 dnode 保持一致，所以您只可以使用 ALTER ALL DNODES 来更新全部 dnodes 的全局配置参数。 + +配置参数是否可以动态修改，有以下三种情况： +1. 支持动态修改 立即生效 +2. 支持动态修改 重启生效 +3. 不支持动态修改 + +对于重启后生效的配置参数，您可以通过 SHOW VARIABLES 或 SHOW DNODE dnode_id VARIABLE 看到修改后的值，但是需要重启数据库服务才使其生效。 + ## 添加管理节点 ```sql @@ -135,20 +126,10 @@ SHOW CLUSTER ALIVE; 如果将客户端也看作广义的集群的一部分，可以通过如下命令动态修改客户端配置参数。 -```sql -ALTER LOCAL local_option +您可以使用以上语法更该客户端的配置参数，并且不需要重启客户端，修改后立即生效。 -local_option: { - 'resetLog' - | 'rpcDebugFlag' 'value' - | 'tmrDebugFlag' 'value' - | 'cDebugFlag' 'value' - | 'uDebugFlag' 'value' - | 'debugFlag' 'value' -} -``` +对于一个配置参数是否支持动态修改，请您参考以下页面：[taosc 参考手册](../01-components/02-taosc.md) -上面语法中的参数与在配置文件中配置客户端的用法相同，但不需要重启客户端，修改后立即生效。 ## 查看客户端配置 diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 9e5d0290cd..3b12922129 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -530,7 +530,7 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input int32_t taosAddClientLogCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddDir(pCfg, "configDir", configDir, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddDir(pCfg, "scriptDir", configDir, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddDir(pCfg, "logDir", tsLogDir, CFG_SCOPE_BOTH, CFG_DYN_BOTH_LAZY, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddDir(pCfg, "logDir", tsLogDir, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN( cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfLogLines", tsNumOfLogLines, 1000, 2000000000, CFG_SCOPE_BOTH, @@ -673,24 +673,24 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { cfgAddInt32(pCfg, "bypassFlag", tsBypassFlag, 0, INT32_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_LOCAL)); tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH, CFG_DYN_BOTH, + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH, CFG_DYN_BOTH_LAZY, CFG_CATEGORY_GLOBAL)); tsNumOfRpcSessions = TRANGE(tsNumOfRpcSessions, 100, 100000); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_BOTH, - CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH, + CFG_DYN_BOTH_LAZY, CFG_CATEGORY_GLOBAL)); tsShareConnLimit = TRANGE(tsShareConnLimit, 1, 512); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "shareConnLimit", tsShareConnLimit, 1, 512, CFG_SCOPE_BOTH, CFG_DYN_SERVER_LAZY, + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "shareConnLimit", tsShareConnLimit, 1, 512, CFG_SCOPE_BOTH, CFG_DYN_BOTH_LAZY, CFG_CATEGORY_GLOBAL)); tsReadTimeout = TRANGE(tsReadTimeout, 64, 24 * 3600 * 7); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "readTimeout", tsReadTimeout, 64, 24 * 3600 * 7, CFG_SCOPE_BOTH, CFG_DYN_NONE, - CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "readTimeout", tsReadTimeout, 64, 24 * 3600 * 7, CFG_SCOPE_BOTH, + CFG_DYN_BOTH_LAZY, CFG_CATEGORY_GLOBAL)); tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 10000000); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "timeToGetAvailableConn", tsTimeToGetAvailableConn, 20, 1000000, CFG_SCOPE_BOTH, - CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); + CFG_DYN_BOTH_LAZY, CFG_CATEGORY_GLOBAL)); tsNumOfTaskQueueThreads = tsNumOfCores * 2; tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 16); @@ -785,14 +785,14 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { // clang-format off TAOS_CHECK_RETURN(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER, CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "encryptAlgorithm", tsEncryptAlgorithm, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "encryptScope", tsEncryptScope, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "encryptAlgorithm", tsEncryptAlgorithm, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "encryptScope", tsEncryptScope, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); @@ -810,9 +810,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16, (TSDB_SYNC_SNAP_BUFFER_SIZE >> 2), CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "syncLogBufferMemoryAllowed", tsLogBufferMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); @@ -822,7 +822,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 86400, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); @@ -833,7 +833,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddString(pCfg, "slowLogExceptDb", tsSlowLogExceptDb, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitorLogProtocol", tsMonitorLogProtocol, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); @@ -845,11 +845,11 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "auditInterval", tsAuditInterval, 500, 200000, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "telemetryInterval", tsTelemInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_BOTH,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "telemetryInterval", tsTelemInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_SERVER, CFG_DYN_BOTH,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "rsyncPort", tsRsyncPort, 1, 65535, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "rsyncPort", tsRsyncPort, 1, 65535, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "snodeAddress", tsSnodeAddress, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); @@ -896,9 +896,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsFPrecision = pItem->fval; TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "curRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "ifAdtFse", tsIfAdtFse, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "compressor", tsCompressor, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "curRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "ifAdtFse", tsIfAdtFse, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "compressor", tsCompressor, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_LOCAL)); @@ -2315,10 +2315,8 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"slowLogThreshold", &tsSlowLogThreshold}, {"compressMsgSize", &tsCompressMsgSize}, {"compressor", &tsCompressor}, - {"curRange", &tsCurRange}, {"dPrecision", &tsDPrecision}, {"fPrecision", &tsFPrecision}, - {"ifAdtFse", &tsIfAdtFse}, {"maxRange", &tsMaxRange}, {"maxTsmaNum", &tsMaxTsmaNum}, {"queryRsmaTolerance", &tsQueryRsmaTolerance}, @@ -2335,7 +2333,6 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"randErrorScope", &tsRandErrScope}, {"syncLogBufferMemoryAllowed", &tsLogBufferMemoryAllowed}, {"resolveFQDNRetryTime", &tsResolveFQDNRetryTime}, - {"syncElectInterval", &tsElectInterval}, {"syncHeartbeatInterval", &tsHeartbeatInterval}, {"syncHeartbeatTimeout", &tsHeartbeatTimeout}, {"syncSnapReplMaxWaitN", &tsSnapReplMaxWaitN}, @@ -2369,9 +2366,6 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"ttlFlushThreshold", &tsTtlFlushThreshold}, {"ttlPushInterval", &tsTtlPushIntervalSec}, {"ttlUnit", &tsTtlUnit}, - {"udf", &tsStartUdfd}, - {"udfdLdLibPath", &tsUdfdLdLibPath}, - {"udfdResFuncs", &tsUdfdResFuncs}, {"s3Accesskey", &tsS3AccessKey}, {"s3BucketName", &tsS3BucketName}, {"s3Endpoint", &tsS3Endpoint}, @@ -2384,17 +2378,16 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"mndLogRetention", &tsMndLogRetention}, {"supportVnodes", &tsNumOfSupportVnodes}, {"experimental", &tsExperimental}, - {"maxTsmaNum", &tsMaxTsmaNum}, - {"maxShellConns", &tsMaxShellConns}, {"numOfRpcSessions", &tsNumOfRpcSessions}, - {"numOfRpcThreads", &tsNumOfRpcThreads}, {"rpcQueueMemoryAllowed", &tsQueueMemoryAllowed}, {"shellActivityTimer", &tsShellActivityTimer}, - {"timeToGetAvailableConn", &tsTimeToGetAvailableConn}, {"readTimeout", &tsReadTimeout}, {"safetyCheckLevel", &tsSafetyCheckLevel}, - {"bypassFlag", &tsBypassFlag}}; + {"bypassFlag", &tsBypassFlag}, + {"arbHeartBeatIntervalSec", &tsArbHeartBeatIntervalSec}, + {"arbCheckSyncIntervalSec", &tsArbCheckSyncIntervalSec}, + {"arbSetAssignedTimeoutSec", &tsArbSetAssignedTimeoutSec}}; if ((code = taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true)) != TSDB_CODE_SUCCESS) { code = taosCfgSetOption(options, tListLen(options), pItem, false); @@ -2643,6 +2636,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { {"multiResultFunctionStarReturnTags", &tsMultiResultFunctionStarReturnTags}, {"maxTsmaCalcDelay", &tsMaxTsmaCalcDelay}, {"tsmaDataDeleteMark", &tsmaDataDeleteMark}, + {"numOfRpcSessions", &tsNumOfRpcSessions}, {"bypassFlag", &tsBypassFlag}}; if ((code = taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true)) != TSDB_CODE_SUCCESS) { From e68cff042769a5fc8383fcd74a8cce6bdd841b22 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 9 Dec 2024 17:11:03 +0800 Subject: [PATCH 69/99] Add config param diskIDCheckEnabled. --- docs/zh/14-reference/01-components/01-taosd.md | 1 + include/common/tglobal.h | 1 + source/common/src/tglobal.c | 8 ++++++++ 3 files changed, 10 insertions(+) diff --git a/docs/zh/14-reference/01-components/01-taosd.md b/docs/zh/14-reference/01-components/01-taosd.md index c70393faa3..7428495691 100644 --- a/docs/zh/14-reference/01-components/01-taosd.md +++ b/docs/zh/14-reference/01-components/01-taosd.md @@ -164,6 +164,7 @@ charset 的有效值是 UTF-8。 |参数名称|支持版本|动态修改|参数含义| |--------------------|----------|-------------------------|-| |dataDir | |不支持动态修改 |数据文件目录，所有的数据文件都将写入该目录，默认值 /var/lib/taos| +|diskIDCheckEnabled | |不支持动态修改 |在 3.3.4.3 后，在重启 dnode 时增加了检查 dataDir 所在磁盘 id 是否发生改变，0:进行检查，1:不进行检查；默认值：1| |tempDir | |不支持动态修改 |指定所有系统运行过程中的临时文件生成的目录，默认值 /tmp| |minimalDataDirGB | |不支持动态修改 |dataDir 指定的时序数据存储目录所需要保留的最小空间，单位 GB，默认值 2| |minimalTmpDirGB | |不支持动态修改 |tempDir 所指定的临时文件目录所需要保留的最小空间，单位 GB，默认值 1| diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 5f5805b4b9..dbce6826a3 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -249,6 +249,7 @@ extern int64_t tsmaDataDeleteMark; extern int64_t tsWalFsyncDataSizeLimit; // internal +extern bool tsDiskIDCheckEnabled; extern int32_t tsTransPullupInterval; extern int32_t tsCompactPullupInterval; extern int32_t tsMqRebalanceInterval; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 3b12922129..0c5c762bcd 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -286,6 +286,7 @@ int32_t tsTtlFlushThreshold = 100; /* maximum number of dirty items in memory. int32_t tsTtlBatchDropNum = 10000; // number of tables dropped per batch // internal +bool tsDiskIDCheckEnabled = true; int32_t tsTransPullupInterval = 2; int32_t tsCompactPullupInterval = 10; int32_t tsMqRebalanceInterval = 2; @@ -858,6 +859,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tmqRowSize", tmqRowSize, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxTsmaNum", tsMaxTsmaNum, 0, 3, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "diskIDCheckEnabled", tsDiskIDCheckEnabled, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); @@ -1634,6 +1636,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "maxTsmaNum"); tsMaxTsmaNum = pItem->i32; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "diskIDCheckEnabled"); + tsDiskIDCheckEnabled = pItem->bval; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "transPullupInterval"); tsTransPullupInterval = pItem->i32; @@ -1969,6 +1974,9 @@ int32_t cfgDeserialize(SArray *array, char *buf, bool isGlobal) { continue; } if (strcasecmp(pItem->name, "dataDir") == 0) { + if (!tsDiskIDCheckEnabled) { + continue; + } int sz = cJSON_GetArraySize(pJson); cJSON *filed = NULL; // check disk id for each dir From 23415cc6d7eb8a828d4128b0bd748261938666a5 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 9 Dec 2024 19:07:27 +0800 Subject: [PATCH 70/99] Fix some review errors. --- include/common/tmsg.h | 2 +- source/common/src/msg/tmsg.c | 2 ++ source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 6 +++- source/dnode/mnode/impl/src/mndConfig.c | 33 ++++----------------- 4 files changed, 14 insertions(+), 29 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 8d17dd48ab..cec63192d0 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1835,7 +1835,7 @@ typedef struct { int32_t tSerializeSConfigReq(void* buf, int32_t bufLen, SConfigReq* pReq); int32_t tDeserializeSConfigReq(void* buf, int32_t bufLen, SConfigReq* pReq); -void tFreeSStatusReq(SStatusReq* pReq); +void tFreeSConfigReq(SConfigReq* pReq); typedef struct { int32_t dnodeId; diff --git a/source/common/src/msg/tmsg.c b/source/common/src/msg/tmsg.c index 232845f0c7..aa53b4a022 100644 --- a/source/common/src/msg/tmsg.c +++ b/source/common/src/msg/tmsg.c @@ -1644,6 +1644,8 @@ _exit: return code; } +void tFreeSConfigReq(SConfigReq *pReq) { taosArrayDestroy(pReq->array); } + int32_t tSerializeSConfigRsp(void *buf, int32_t bufLen, SConfigRsp *pRsp) { SEncoder encoder = {0}; int32_t code = 0; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index a856a723c6..3831dd2a2d 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -287,6 +287,7 @@ static void dmProcessConfigRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { const STraceId *trace = &pRsp->info.traceId; int32_t code = 0; SConfigRsp configRsp = {0}; + bool needStop = false; if (pRsp->code != 0) { if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pMgmt->pData->dropped && pMgmt->pData->dnodeId > 0) { @@ -315,7 +316,7 @@ static void dmProcessConfigRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { } else { // log the difference configurations printConfigNotMatch(configRsp.array); - dmStop(); + needStop = true; goto _exit; } } @@ -347,6 +348,9 @@ static void dmProcessConfigRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { _exit: tFreeSConfigRsp(&configRsp); rpcFreeCont(pRsp->pCont); + if (needStop) { + dmStop(); + } } void dmSendConfigReq(SDnodeMgmt *pMgmt) { diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index b283c0f221..0a9c51511f 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -38,7 +38,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq); static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessConfigReq(SRpcMsg *pReq); static int32_t mndInitWriteCfg(SMnode *pMnode); -static int32_t mndInitReadCfg(SMnode *pMnode); +static int32_t mndMaybeReBuildCfg(SMnode *pMnode); static int32_t initConfigArrayFromSdb(SMnode *pMnode, SArray *array); static void cfgArrayCleanUp(SArray *array); @@ -219,7 +219,7 @@ static int32_t mndCfgActionUpdate(SSdb *pSdb, SConfigObj *pOld, SConfigObj *pNew static int32_t mndCfgActionDeploy(SMnode *pMnode) { return mndInitWriteCfg(pMnode); } -static int32_t mndCfgActionPrepare(SMnode *pMnode) { return mndInitReadCfg(pMnode); } +static int32_t mndCfgActionPrepare(SMnode *pMnode) { return mndMaybeReBuildCfg(pMnode); } static int32_t mndProcessConfigReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; @@ -322,14 +322,9 @@ _OVER: return TSDB_CODE_SUCCESS; } -int32_t mndInitReadCfg(SMnode *pMnode) { - int32_t code = 0; - int32_t sz = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "init-read-config"); - if (pTrans == NULL) { - mError("failed to init read cfg in create trans, since %s", terrstr()); - goto _OVER; - } +int32_t mndMaybeReBuildCfg(SMnode *pMnode) { + int32_t code = 0; + int32_t sz = -1; SConfigObj *obj = sdbAcquire(pMnode->pSdb, SDB_CFG, "tsmmConfigVersion"); if (obj == NULL) { code = mndInitWriteCfg(pMnode); @@ -339,24 +334,8 @@ int32_t mndInitReadCfg(SMnode *pMnode) { mInfo("failed to acquire mnd config version, try to rebuild it , since %s", terrstr()); goto _OVER; } - - sz = taosArrayGetSize(taosGetGlobalCfg(tsCfg)); - for (int i = 0; i < sz; ++i) { - SConfigItem *item = taosArrayGet(taosGetGlobalCfg(tsCfg), i); - SConfigObj *newObj = sdbAcquire(pMnode->pSdb, SDB_CFG, item->name); - if (newObj == NULL) { - mInfo("failed to acquire mnd config:%s, since %s", item->name, terrstr()); - continue; - } - code = cfgUpdateItem(item, newObj); - if (code != 0) { - mError("failed to update mnd config:%s, since %s", item->name, terrstr()); - } - sdbRelease(pMnode->pSdb, newObj); - } _OVER: - mndTransDrop(pTrans); - return TSDB_CODE_SUCCESS; + return code; } int32_t mndSetCreateConfigCommitLogs(STrans *pTrans, SConfigObj *item) { From 37d6af96b0c31aa9d97bc2e5ba55a606e75c5859 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 9 Dec 2024 19:36:15 +0800 Subject: [PATCH 71/99] enh some codes. --- source/dnode/mnode/impl/src/mndConfig.c | 129 +++++++++++++++++++++++- source/dnode/mnode/impl/src/mndDnode.c | 117 --------------------- 2 files changed, 124 insertions(+), 122 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index 0a9c51511f..13468d2792 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -32,16 +32,18 @@ enum CfgAlterType { static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pInMCfgReq, int32_t optLen, int32_t *pOutValue); static int32_t cfgUpdateItem(SConfigItem *pItem, SConfigObj *obj); -static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue, ECfgDataType dtype, - int32_t tsmmConfigVersion); +static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq); static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq); static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessConfigReq(SRpcMsg *pReq); static int32_t mndInitWriteCfg(SMnode *pMnode); -static int32_t mndMaybeReBuildCfg(SMnode *pMnode); +static int32_t mndInitReadCfg(SMnode *pMnode); static int32_t initConfigArrayFromSdb(SMnode *pMnode, SArray *array); static void cfgArrayCleanUp(SArray *array); +static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue, ECfgDataType dtype, + int32_t tsmmConfigVersion); + int32_t mndSetCreateConfigCommitLogs(STrans *pTrans, SConfigObj *obj); int32_t mndInitConfig(SMnode *pMnode) { @@ -59,6 +61,7 @@ int32_t mndInitConfig(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_CONFIG, mndProcessConfigReq); mndSetMsgHandle(pMnode, TDMT_MND_CONFIG_DNODE, mndProcessConfigDnodeReq); mndSetMsgHandle(pMnode, TDMT_DND_CONFIG_DNODE_RSP, mndProcessConfigDnodeRsp); + mndSetMsgHandle(pMnode, TDMT_MND_SHOW_VARIABLES, mndProcessShowVariablesReq); return sdbSetTable(pMnode->pSdb, table); } @@ -219,7 +222,7 @@ static int32_t mndCfgActionUpdate(SSdb *pSdb, SConfigObj *pOld, SConfigObj *pNew static int32_t mndCfgActionDeploy(SMnode *pMnode) { return mndInitWriteCfg(pMnode); } -static int32_t mndCfgActionPrepare(SMnode *pMnode) { return mndMaybeReBuildCfg(pMnode); } +static int32_t mndCfgActionPrepare(SMnode *pMnode) { return mndInitReadCfg(pMnode); } static int32_t mndProcessConfigReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; @@ -322,7 +325,7 @@ _OVER: return TSDB_CODE_SUCCESS; } -int32_t mndMaybeReBuildCfg(SMnode *pMnode) { +int32_t mndInitReadCfg(SMnode *pMnode) { int32_t code = 0; int32_t sz = -1; SConfigObj *obj = sdbAcquire(pMnode->pSdb, SDB_CFG, "tsmmConfigVersion"); @@ -679,4 +682,120 @@ static void cfgArrayCleanUp(SArray *array) { } taosArrayDestroy(array); +} + +SArray *initVariablesFromItems(SArray *pItems) { + if (pItems == NULL) { + return NULL; + } + + int32_t sz = taosArrayGetSize(pItems); + + SArray *pInfos = taosArrayInit(sz, sizeof(SVariablesInfo)); + for (int32_t i = 0; i < sz; ++i) { + SConfigItem *pItem = taosArrayGet(pItems, i); + SVariablesInfo info = {0}; + strcpy(info.name, pItem->name); + + // init info value + switch (pItem->dtype) { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + sprintf(info.value, "%d", pItem->bval); + break; + case CFG_DTYPE_INT32: + sprintf(info.value, "%d", pItem->i32); + break; + case CFG_DTYPE_INT64: + sprintf(info.value, "%" PRId64, pItem->i64); + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + sprintf(info.value, "%f", pItem->fval); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + sprintf(info.value, "%s", pItem->str); + break; + } + + // init info scope + switch (pItem->scope) { + case CFG_SCOPE_SERVER: + strcpy(info.scope, "server"); + break; + case CFG_SCOPE_CLIENT: + strcpy(info.scope, "client"); + break; + case CFG_SCOPE_BOTH: + strcpy(info.scope, "both"); + break; + default: + strcpy(info.scope, "unknown"); + break; + } + // init info category + switch (pItem->category) { + case CFG_CATEGORY_GLOBAL: + strcpy(info.category, "global"); + break; + case CFG_CATEGORY_LOCAL: + strcpy(info.category, "local"); + break; + default: + strcpy(info.category, "unknown"); + break; + } + if (NULL == taosArrayPush(pInfos, &info)) { + mError("failed to push info to array while init variables from items,since %s", tstrerror(terrno)); + return NULL; + } + } + + return pInfos; +} + +static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { + SShowVariablesRsp rsp = {0}; + int32_t code = -1; + + if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_SHOW_VARIABLES) != 0) { + goto _OVER; + } + + SVariablesInfo info = {0}; + + rsp.variables = initVariablesFromItems(taosGetGlobalCfg(tsCfg)); + if (rsp.variables == NULL) { + code = terrno; + goto _OVER; + } + int32_t rspLen = tSerializeSShowVariablesRsp(NULL, 0, &rsp); + void *pRsp = rpcMallocCont(rspLen); + if (pRsp == NULL) { + code = terrno; + goto _OVER; + } + + if ((rspLen = tSerializeSShowVariablesRsp(pRsp, rspLen, &rsp)) <= 0) { + code = rspLen; + goto _OVER; + } + + pReq->info.rspLen = rspLen; + pReq->info.rsp = pRsp; + code = 0; + +_OVER: + + if (code != 0) { + mError("failed to get show variables info since %s", tstrerror(code)); + } + + tFreeSShowVariablesRsp(&rsp); + TAOS_RETURN(code); } \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index dfe0c1e933..fd520f723b 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -77,7 +77,6 @@ static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode); static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode); static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOld, SDnodeObj *pNew); static int32_t mndProcessDnodeListReq(SRpcMsg *pReq); -static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq); static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq); static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq); @@ -118,7 +117,6 @@ int32_t mndInitDnode(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_STATUS, mndProcessStatusReq); mndSetMsgHandle(pMnode, TDMT_MND_NOTIFY, mndProcessNotifyReq); mndSetMsgHandle(pMnode, TDMT_MND_DNODE_LIST, mndProcessDnodeListReq); - mndSetMsgHandle(pMnode, TDMT_MND_SHOW_VARIABLES, mndProcessShowVariablesReq); mndSetMsgHandle(pMnode, TDMT_MND_RESTORE_DNODE, mndProcessRestoreDnodeReq); mndSetMsgHandle(pMnode, TDMT_MND_STATIS, mndProcessStatisReq); mndSetMsgHandle(pMnode, TDMT_MND_AUDIT, mndProcessAuditReq); @@ -1079,121 +1077,6 @@ static void getSlowLogScopeString(int32_t scope, char *result) { } } -SArray *initVariablesFromItems(SArray *pItems) { - if (pItems == NULL) { - return NULL; - } - - int32_t sz = taosArrayGetSize(pItems); - - SArray *pInfos = taosArrayInit(sz, sizeof(SVariablesInfo)); - for (int32_t i = 0; i < sz; ++i) { - SConfigItem *pItem = taosArrayGet(pItems, i); - SVariablesInfo info = {0}; - strcpy(info.name, pItem->name); - - // init info value - switch (pItem->dtype) { - case CFG_DTYPE_NONE: - break; - case CFG_DTYPE_BOOL: - sprintf(info.value, "%d", pItem->bval); - break; - case CFG_DTYPE_INT32: - sprintf(info.value, "%d", pItem->i32); - break; - case CFG_DTYPE_INT64: - sprintf(info.value, "%" PRId64, pItem->i64); - break; - case CFG_DTYPE_FLOAT: - case CFG_DTYPE_DOUBLE: - sprintf(info.value, "%f", pItem->fval); - break; - case CFG_DTYPE_STRING: - case CFG_DTYPE_DIR: - case CFG_DTYPE_LOCALE: - case CFG_DTYPE_CHARSET: - case CFG_DTYPE_TIMEZONE: - sprintf(info.value, "%s", pItem->str); - break; - } - - // init info scope - switch (pItem->scope) { - case CFG_SCOPE_SERVER: - strcpy(info.scope, "server"); - break; - case CFG_SCOPE_CLIENT: - strcpy(info.scope, "client"); - break; - case CFG_SCOPE_BOTH: - strcpy(info.scope, "both"); - break; - default: - strcpy(info.scope, "unknown"); - break; - } - // init info category - switch (pItem->category) { - case CFG_CATEGORY_GLOBAL: - strcpy(info.category, "global"); - break; - case CFG_CATEGORY_LOCAL: - strcpy(info.category, "local"); - break; - default: - strcpy(info.category, "unknown"); - break; - } - if (NULL == taosArrayPush(pInfos, &info)) { - mError("failed to push info to array while init variables from items,since %s", tstrerror(terrno)); - return NULL; - } - } - - return pInfos; -} - -static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { - SShowVariablesRsp rsp = {0}; - int32_t code = -1; - - if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_SHOW_VARIABLES) != 0) { - goto _OVER; - } - - SVariablesInfo info = {0}; - - rsp.variables = initVariablesFromItems(taosGetGlobalCfg(tsCfg)); - if (rsp.variables == NULL) { - code = terrno; - goto _OVER; - } - int32_t rspLen = tSerializeSShowVariablesRsp(NULL, 0, &rsp); - void *pRsp = rpcMallocCont(rspLen); - if (pRsp == NULL) { - code = terrno; - goto _OVER; - } - - if ((rspLen = tSerializeSShowVariablesRsp(pRsp, rspLen, &rsp)) <= 0) { - code = rspLen; - goto _OVER; - } - - pReq->info.rspLen = rspLen; - pReq->info.rsp = pRsp; - code = 0; - -_OVER: - - if (code != 0) { - mError("failed to get show variables info since %s", tstrerror(code)); - } - - tFreeSShowVariablesRsp(&rsp); - TAOS_RETURN(code); -} static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; From 1ced6069eed58964d5b2deb10d073d0e4ae2f900 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 10 Dec 2024 10:39:50 +0800 Subject: [PATCH 72/99] Fix some review problems. --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 2 +- source/dnode/mnode/impl/src/mndConfig.c | 77 ++++++++++++++++----- 3 files changed, 60 insertions(+), 21 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 3831dd2a2d..24a79b4440 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -508,7 +508,7 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { if (cfgReq.version > 0) { tsdmConfigVersion = cfgReq.version; } - return TSDB_CODE_SUCCESS; + return code; } int32_t dmProcessCreateEncryptKeyReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index d160f6584c..982866030a 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -342,7 +342,7 @@ int32_t dmStartConfigThread(SDnodeMgmt *pMgmt) { int32_t code = 0; TdThreadAttr thAttr; (void)taosThreadAttrInit(&thAttr); - (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_DETACHED); if (taosThreadCreate(&pMgmt->configThread, &thAttr, dmConfigThreadFp, pMgmt) != 0) { code = TAOS_SYSTEM_ERROR(errno); dError("failed to create config thread since %s", tstrerror(code)); diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index 13468d2792..0f7b959300 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -78,7 +78,7 @@ SSdbRaw *mnCfgActionEncode(SConfigObj *obj) { int32_t dataPos = 0; char name[CFG_NAME_MAX_LEN] = {0}; - strncpy(name, obj->name, CFG_NAME_MAX_LEN); + tstrncpy(name, obj->name, CFG_NAME_MAX_LEN); SDB_SET_BINARY(pRaw, dataPos, name, CFG_NAME_MAX_LEN, _OVER) SDB_SET_INT32(pRaw, dataPos, obj->dtype, _OVER) switch (obj->dtype) { @@ -274,8 +274,13 @@ static int32_t mndProcessConfigReq(SRpcMsg *pReq) { goto _OVER; } void *pHead = rpcMallocCont(contLen); + if (pHead == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } contLen = tSerializeSConfigRsp(pHead, contLen, &configRsp); if (contLen < 0) { + rpcFreeCont(pHead); code = contLen; goto _OVER; } @@ -283,13 +288,16 @@ static int32_t mndProcessConfigReq(SRpcMsg *pReq) { pReq->info.rsp = pHead; _OVER: + if (code != 0) { + mError("failed to process config req, since %s", tstrerror(code)); + } cfgArrayCleanUp(array); mndReleaseDnode(pMnode, pDnode); return TSDB_CODE_SUCCESS; } int32_t mndInitWriteCfg(SMnode *pMnode) { - int code = -1; + int code = 0; size_t sz = 0; mInfo("init write cfg to sdb"); @@ -313,16 +321,23 @@ int32_t mndInitWriteCfg(SMnode *pMnode) { for (int i = 0; i < sz; ++i) { SConfigItem *item = taosArrayGet(taosGetGlobalCfg(tsCfg), i); SConfigObj *obj = mndInitConfigObj(item); + if (obj == NULL) { + code = terrno; + goto _OVER; + } if ((code = mndSetCreateConfigCommitLogs(pTrans, obj)) != 0) { - mError("failed to init mnd config:%s, since %s", item->name, terrstr()); + mError("failed to init mnd config:%s, since %s", item->name, tstrerror(code)); } taosMemoryFree(obj); } if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER; _OVER: + if (code != 0) { + mError("failed to init write cfg, since %s", tstrerror(code)); + } mndTransDrop(pTrans); - return TSDB_CODE_SUCCESS; + return code; } int32_t mndInitReadCfg(SMnode *pMnode) { @@ -386,6 +401,9 @@ int32_t cfgUpdateItem(SConfigItem *pItem, SConfigObj *obj) { if (obj->str != NULL) { taosMemoryFree(pItem->str); pItem->str = taosStrdup(obj->str); + if (pItem->str == NULL) { + TAOS_RETURN(terrno); + } } break; } @@ -408,7 +426,7 @@ static int32_t mndMCfg2DCfg(SMCfgDnodeReq *pMCfgReq, SDCfgDnodeReq *pDCfgReq) { } size_t optLen = p - pMCfgReq->config; - (void)strncpy(pDCfgReq->config, pMCfgReq->config, optLen); + tstrncpy(pDCfgReq->config, pMCfgReq->config, optLen); pDCfgReq->config[optLen] = 0; if (' ' == pMCfgReq->config[optLen]) { @@ -445,6 +463,8 @@ static int32_t mndSendCfgDnodeReq(SMnode *pMnode, int32_t dnodeId, SDCfgDnodeReq if (pBuf != NULL) { if ((bufLen = tSerializeSDCfgDnodeReq(pBuf, bufLen, pDcfgReq)) <= 0) { + sdbCancelFetch(pMnode->pSdb, pIter); + sdbRelease(pMnode->pSdb, pDnode); code = bufLen; return code; } @@ -483,14 +503,17 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { SDCfgDnodeReq dcfgReq = {0}; if (strcasecmp(cfgReq.config, "resetlog") == 0) { - (void)strcpy(dcfgReq.config, "resetlog"); + tstrncpy(dcfgReq.config, "resetlog", 8); goto _send_req; #ifdef TD_ENTERPRISE } else if (strncasecmp(cfgReq.config, "s3blocksize", 11) == 0) { int32_t optLen = strlen("s3blocksize"); int32_t flag = -1; int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); - if (code < 0) return code; + if (code < 0) { + tFreeSMCfgDnodeReq(&cfgReq); + TAOS_RETURN(code); + } if (flag > 1024 * 1024 || (flag > -1 && flag < 1024) || flag < -1) { mError("dnode:%d, failed to config s3blocksize since value:%d. Valid range: -1 or [1024, 1024 * 1024]", @@ -567,11 +590,11 @@ static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t optLen, int32 if (' ' == pMCfgReq->config[optLen]) { // 'key value' if (strlen(pMCfgReq->value) != 0) goto _err; - *pOutValue = atoi(pMCfgReq->config + optLen + 1); + *pOutValue = taosStr2int32(pMCfgReq->config + optLen + 1); } else { // 'key' 'value' if (strlen(pMCfgReq->value) == 0) goto _err; - *pOutValue = atoi(pMCfgReq->value); + *pOutValue = taosStr2int32(pMCfgReq->value); } TAOS_RETURN(code); @@ -588,15 +611,15 @@ static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pVal int32_t lino = -1; SConfigObj pVersion = {0}, pObj = {0}; - strncpy(pVersion.name, "tsmmConfigVersion", CFG_NAME_MAX_LEN); + tstrncpy(pVersion.name, "tsmmConfigVersion", CFG_NAME_MAX_LEN); pVersion.i32 = tsmmConfigVersion; pVersion.dtype = CFG_DTYPE_INT32; pObj.dtype = dtype; - strncpy(pObj.name, name, CFG_NAME_MAX_LEN); + tstrncpy(pObj.name, name, CFG_NAME_MAX_LEN); TAOS_CHECK_GOTO(mndUpdateObj(&pObj, name, pValue), &lino, _OVER); - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_ARBGROUP, NULL, "update-config"); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "update-config"); if (pTrans == NULL) { if (terrno != 0) code = terrno; goto _OVER; @@ -615,16 +638,17 @@ _OVER: } static int32_t initConfigArrayFromSdb(SMnode *pMnode, SArray *array) { - int32_t code = 0; - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + int32_t code = 0; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + SConfigObj *obj = NULL; + while (1) { - SConfigObj *obj = NULL; pIter = sdbFetch(pSdb, SDB_CFG, pIter, (void **)&obj); if (pIter == NULL) break; if (obj == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - break; + goto _exit; } if (strcasecmp(obj->name, "tsmmConfigVersion") == 0) { continue; @@ -632,6 +656,10 @@ static int32_t initConfigArrayFromSdb(SMnode *pMnode, SArray *array) { SConfigItem item = {0}; item.dtype = obj->dtype; item.name = taosStrdup(obj->name); + if (item.name == NULL) { + code = terrno; + goto _exit; + } switch (obj->dtype) { case CFG_DTYPE_NONE: break; @@ -654,15 +682,25 @@ static int32_t initConfigArrayFromSdb(SMnode *pMnode, SArray *array) { case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: item.str = taosStrdup(obj->str); + if (item.str == NULL) { + code = terrno; + goto _exit; + } break; } if (taosArrayPush(array, &item) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - sdbRelease(pSdb, obj); + goto _exit; break; } sdbRelease(pSdb, obj); } +_exit: + if (code != 0) { + mError("failed to init config array from sdb, since %s", tstrerror(code)); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, obj); + } return code; } @@ -752,6 +790,7 @@ SArray *initVariablesFromItems(SArray *pItems) { } if (NULL == taosArrayPush(pInfos, &info)) { mError("failed to push info to array while init variables from items,since %s", tstrerror(terrno)); + taosArrayDestroy(pInfos); return NULL; } } @@ -763,7 +802,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { SShowVariablesRsp rsp = {0}; int32_t code = -1; - if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_SHOW_VARIABLES) != 0) { + if ((code = mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_SHOW_VARIABLES)) != 0) { goto _OVER; } From 2e1b94ff87567ddd2c80b71e99eaf96a8fdc87de Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 10 Dec 2024 11:33:03 +0800 Subject: [PATCH 73/99] Fix review errors. --- docs/en/14-reference/09-error-code.md | 2 ++ docs/zh/14-reference/09-error-code.md | 1 + include/util/taoserror.h | 1 + source/common/src/msg/tmsg.c | 3 +++ source/common/src/tglobal.c | 39 +++++++++++++++++++++------ source/util/src/terror.c | 1 + 6 files changed, 39 insertions(+), 8 deletions(-) diff --git a/docs/en/14-reference/09-error-code.md b/docs/en/14-reference/09-error-code.md index 46f8687843..bf211156ef 100644 --- a/docs/en/14-reference/09-error-code.md +++ b/docs/en/14-reference/09-error-code.md @@ -72,6 +72,8 @@ This document details the server error codes that may be encountered when using | 0x80000133 | Invalid operation | Invalid or unsupported operation | 1. Modify to confirm the current operation is legal and supported, check parameter validity 2. If the problem persists, preserve the scene and logs, report issue on github | | 0x80000134 | Invalid value | Invalid value | Preserve the scene and logs, report issue on github | | 0x80000135 | Invalid fqdn | Invalid FQDN | Check if the configured or input FQDN value is correct | +| 0x8000013C | Invalid disk id | Invalid disk id | Check users whether the mounted disk is invalid or use the parameter diskIDCheckEnabled to skip the disk check. | + ## tsc diff --git a/docs/zh/14-reference/09-error-code.md b/docs/zh/14-reference/09-error-code.md index 4bd495129e..72d4897763 100644 --- a/docs/zh/14-reference/09-error-code.md +++ b/docs/zh/14-reference/09-error-code.md @@ -75,6 +75,7 @@ description: TDengine 服务端的错误码列表和详细说明 | 0x80000133 | Invalid operation | 无效的或不支持的操作 | 1. 修改确认当前操作为合法有效支持的操作，检查参数有效性 2. 如果问题还未解决，保留现场和日志，github上报issue | | 0x80000134 | Invalid value | 无效值 | 保留现场和日志，github上报issue | | 0x80000135 | Invalid fqdn | 无效FQDN | 检查配置或输入的FQDN值是否正确 | +| 0x8000013C | Invalid disk id | 不合法的disk id | 建议用户检查挂载磁盘是否失效或者使用参数 diskIDCheckEnabled 来跳过磁盘检查 | diff --git a/include/util/taoserror.h b/include/util/taoserror.h index a01bbf66a6..ca349425d2 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -159,6 +159,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_SOCKET_ERROR TAOS_DEF_ERROR_CODE(0, 0x0139) #define TSDB_CODE_UNSUPPORT_OS TAOS_DEF_ERROR_CODE(0, 0x013A) #define TSDB_CODE_TIME_ERROR TAOS_DEF_ERROR_CODE(0, 0x013B) +#define TSDB_CODE_INVALID_DISK_ID TAOS_DEF_ERROR_CODE(0, 0x013C) //client #define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) diff --git a/source/common/src/msg/tmsg.c b/source/common/src/msg/tmsg.c index aa53b4a022..deb1cbf843 100644 --- a/source/common/src/msg/tmsg.c +++ b/source/common/src/msg/tmsg.c @@ -1636,6 +1636,9 @@ int32_t tDeserializeSConfigReq(void *buf, int32_t bufLen, SConfigReq *pReq) { TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->forceReadConfig)); if (pReq->forceReadConfig) { pReq->array = taosArrayInit(128, sizeof(SConfigItem)); + if (pReq->array == NULL) { + TAOS_CHECK_EXIT(terrno); + } TAOS_CHECK_EXIT(tDeserializeSConfigArray(&decoder, pReq->array)); } tEndDecode(&decoder); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 0c5c762bcd..cefdb1b544 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1947,6 +1947,8 @@ _exit: } int32_t cfgDeserialize(SArray *array, char *buf, bool isGlobal) { + int32_t code = TSDB_CODE_SUCCESS; + cJSON *pRoot = cJSON_Parse(buf); if (pRoot == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -1954,8 +1956,8 @@ int32_t cfgDeserialize(SArray *array, char *buf, bool isGlobal) { if (isGlobal) { cJSON *pItem = cJSON_GetObjectItem(pRoot, "version"); if (pItem == NULL) { - cJSON_Delete(pRoot); - return TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; } tsdmConfigVersion = pItem->valueint; } @@ -1963,9 +1965,8 @@ int32_t cfgDeserialize(SArray *array, char *buf, bool isGlobal) { int32_t sz = taosArrayGetSize(array); cJSON *configs = cJSON_GetObjectItem(pRoot, "configs"); if (configs == NULL) { - uError("failed to get configs from json, since %s", cJSON_GetErrorPtr()); - cJSON_Delete(pRoot); - return TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; } for (int i = 0; i < sz; i++) { SConfigItem *pItem = (SConfigItem *)taosArrayGet(array, i); @@ -1982,15 +1983,31 @@ int32_t cfgDeserialize(SArray *array, char *buf, bool isGlobal) { // check disk id for each dir for (int j = 0; j < sz; j++) { cJSON *diskCfgJson = cJSON_GetArrayItem(pJson, j); + if (diskCfgJson == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + filed = cJSON_GetObjectItem(diskCfgJson, "dir"); + if (filed == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + char *dir = cJSON_GetStringValue(filed); filed = cJSON_GetObjectItem(diskCfgJson, "disk_id"); + if (filed == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + int64_t actDiskID = 0; - int64_t expDiskID = atoll(cJSON_GetStringValue(filed)); + int64_t expDiskID = taosStr2Int64(cJSON_GetStringValue(filed), NULL, 10); if (!taosCheckFileDiskID(dir, &actDiskID, expDiskID)) { uError("failed to check disk id for dir:%s, actDiskID%" PRId64 ", expDiskID%" PRId64, dir, actDiskID, expDiskID); - return TSDB_CODE_FAILED; + code = TSDB_CODE_INVALID_DISK_ID; + goto _exit; } } continue; @@ -2020,13 +2037,19 @@ int32_t cfgDeserialize(SArray *array, char *buf, bool isGlobal) { case CFG_DTYPE_TIMEZONE: taosMemoryFree(pItem->str); pItem->str = taosStrdup(pJson->valuestring); + if (pItem->str == NULL) { + code = terrno; + } break; } } } _exit: + if (code != TSDB_CODE_SUCCESS) { + uError("failed to deserialize config since %s", tstrerror(code)); + } cJSON_Delete(pRoot); - return TSDB_CODE_SUCCESS; + return code; } int32_t readCfgFile(const char *path, bool isGlobal) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index d2be62f56a..f1cb8fcb0b 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -118,6 +118,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MSG_PREPROCESSED, "Message has been proc TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_BUFFER, "Out of buffer") TAOS_DEFINE_ERROR(TSDB_CODE_INTERNAL_ERROR, "Internal error") TAOS_DEFINE_ERROR(TSDB_CODE_TIME_ERROR, "Internal error in time") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_DISK_ID, "Internal error invalid disk id") //client TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_OPERATION, "Invalid operation") From 430cf9a8064f5058b0218b509ee89d6cd4c2a7a6 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 10 Dec 2024 14:07:31 +0800 Subject: [PATCH 74/99] Fix some review errors. --- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 10 ------ source/dnode/mnode/impl/src/mndConfig.c | 35 +++++++++++---------- 2 files changed, 18 insertions(+), 27 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 982866030a..b481a3fc5e 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -51,10 +51,7 @@ static void *dmConfigThreadFp(void *param) { SDnodeMgmt *pMgmt = param; int64_t lastTime = taosGetTimestampMs(); setThreadName("dnode-config"); - int32_t upTimeCount = 0; - int64_t upTime = 0; - while (1) { taosMsleep(200); if (pMgmt->pData->dropped || pMgmt->pData->stopped || tsConfigInited) break; @@ -64,15 +61,8 @@ static void *dmConfigThreadFp(void *param) { float interval = (curTime - lastTime) / 1000.0f; if (interval >= tsStatusInterval) { dmSendConfigReq(pMgmt); - lastTime = curTime; - - if ((upTimeCount = ((upTimeCount + 1) & 63)) == 0) { - upTime = taosGetOsUptime() - tsDndStartOsUptime; - tsDndUpTime = TMAX(tsDndUpTime, upTime); - } } } - return NULL; } diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index 0f7b959300..2f547896d4 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -490,15 +490,14 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { SMCfgDnodeReq cfgReq = {0}; SConfigObj *vObj = sdbAcquire(pMnode->pSdb, SDB_CFG, "tsmmConfigVersion"); if (vObj == NULL) { - mInfo("failed to acquire mnd config version, since %s", terrstr()); - TAOS_RETURN(terrno); + goto _err_out; } + TAOS_CHECK_RETURN(tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq)); int8_t updateIpWhiteList = 0; mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value); if ((code = mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE)) != 0) { - tFreeSMCfgDnodeReq(&cfgReq); - TAOS_RETURN(code); + goto _err_out; } SDCfgDnodeReq dcfgReq = {0}; @@ -511,19 +510,17 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { int32_t flag = -1; int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); if (code < 0) { - tFreeSMCfgDnodeReq(&cfgReq); - TAOS_RETURN(code); + goto _err_out; } if (flag > 1024 * 1024 || (flag > -1 && flag < 1024) || flag < -1) { mError("dnode:%d, failed to config s3blocksize since value:%d. Valid range: -1 or [1024, 1024 * 1024]", cfgReq.dnodeId, flag); code = TSDB_CODE_INVALID_CFG; - tFreeSMCfgDnodeReq(&cfgReq); - TAOS_RETURN(code); + goto _err_out; } - strcpy(dcfgReq.config, "s3blocksize"); + tstrncpy(dcfgReq.config, "s3blocksize", 11); snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); #endif } else { @@ -560,18 +557,22 @@ _send_req : auditRecord(pReq, pMnode->clusterId, "alterDnode", obj, "", cfgReq.sql, cfgReq.sqlLen); } - - tFreeSMCfgDnodeReq(&cfgReq); - dcfgReq.version = vObj->i32; code = mndSendCfgDnodeReq(pMnode, cfgReq.dnodeId, &dcfgReq); - + if (code != 0) { + mError("failed to send config req to dnode:%d, since %s", cfgReq.dnodeId, tstrerror(code)); + goto _err_out; + } // dont care suss or succ; if (updateIpWhiteList) mndRefreshUserIpWhiteList(pMnode); + tFreeSMCfgDnodeReq(&cfgReq); + sdbRelease(pMnode->pSdb, vObj); TAOS_RETURN(code); _err_out: + mError("failed to process config dnode req, since %s", tstrerror(code)); tFreeSMCfgDnodeReq(&cfgReq); + sdbRelease(pMnode->pSdb, vObj); TAOS_RETURN(code); } @@ -600,8 +601,7 @@ static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t optLen, int32 TAOS_RETURN(code); _err: - mError("dnode:%d, failed to config since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config); - code = TSDB_CODE_INVALID_CFG; + mError(" failed to set config since:%s", tstrerror(code)); TAOS_RETURN(code); } @@ -658,6 +658,7 @@ static int32_t initConfigArrayFromSdb(SMnode *pMnode, SArray *array) { item.name = taosStrdup(obj->name); if (item.name == NULL) { code = terrno; + sdbCancelFetch(pSdb, pIter); goto _exit; } switch (obj->dtype) { @@ -683,12 +684,14 @@ static int32_t initConfigArrayFromSdb(SMnode *pMnode, SArray *array) { case CFG_DTYPE_TIMEZONE: item.str = taosStrdup(obj->str); if (item.str == NULL) { + sdbCancelFetch(pSdb, pIter); code = terrno; goto _exit; } break; } if (taosArrayPush(array, &item) == NULL) { + sdbCancelFetch(pSdb, pIter); code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; break; @@ -698,8 +701,6 @@ static int32_t initConfigArrayFromSdb(SMnode *pMnode, SArray *array) { _exit: if (code != 0) { mError("failed to init config array from sdb, since %s", tstrerror(code)); - sdbCancelFetch(pSdb, pIter); - sdbRelease(pSdb, obj); } return code; } From db2a1a84d329e2ac45098ac195a6230d35669a3b Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 10 Dec 2024 14:29:12 +0800 Subject: [PATCH 75/99] Fix build errors. --- source/dnode/mnode/impl/src/mndConfig.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index 2f547896d4..1155495fd4 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -426,17 +426,17 @@ static int32_t mndMCfg2DCfg(SMCfgDnodeReq *pMCfgReq, SDCfgDnodeReq *pDCfgReq) { } size_t optLen = p - pMCfgReq->config; - tstrncpy(pDCfgReq->config, pMCfgReq->config, optLen); + tstrncpy(pDCfgReq->config, pMCfgReq->config, optLen + 1); pDCfgReq->config[optLen] = 0; if (' ' == pMCfgReq->config[optLen]) { // 'key value' if (strlen(pMCfgReq->value) != 0) goto _err; - (void)strcpy(pDCfgReq->value, p + 1); + tstrncpy(pDCfgReq->value, p + 1, strlen(p + 1)); } else { // 'key' 'value' if (strlen(pMCfgReq->value) == 0) goto _err; - (void)strcpy(pDCfgReq->value, pMCfgReq->value); + tstrncpy(pDCfgReq->value, pMCfgReq->value, strlen(pMCfgReq->value)); } TAOS_RETURN(code); @@ -591,11 +591,11 @@ static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t optLen, int32 if (' ' == pMCfgReq->config[optLen]) { // 'key value' if (strlen(pMCfgReq->value) != 0) goto _err; - *pOutValue = taosStr2int32(pMCfgReq->config + optLen + 1); + *pOutValue = taosStr2Int32(pMCfgReq->config + optLen + 1, NULL, 10); } else { // 'key' 'value' if (strlen(pMCfgReq->value) == 0) goto _err; - *pOutValue = taosStr2int32(pMCfgReq->value); + *pOutValue = taosStr2Int32(pMCfgReq->value, NULL, 10); } TAOS_RETURN(code); From 1455bb2a41745ba05d355d27bb09af494440ae6e Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 10 Dec 2024 14:43:58 +0800 Subject: [PATCH 76/99] Fix some review errors. --- source/common/src/tglobal.c | 55 ++++++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 59eee06809..bab37d72c6 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -335,7 +335,6 @@ bool tsExperimental = true; int32_t tsMaxTsmaNum = 3; int32_t tsMaxTsmaCalcDelay = 600; int64_t tsmaDataDeleteMark = 1000 * 60 * 60 * 24; // in ms, default to 1d -void* pTimezoneNameMap = NULL; #define TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, pName) \ if ((pItem = cfgGetItem(pCfg, pName)) == NULL) { \ @@ -1419,6 +1418,34 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { static int32_t taosSetSystemCfg(SConfig *pCfg) { SConfigItem *pItem = NULL; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "timezone"); + if (0 == strlen(pItem->str)) { + uError("timezone is not set"); + } else { + TAOS_CHECK_RETURN(osSetTimezone(pItem->str)); + uDebug("timezone format changed from %s to %s", pItem->str, tsTimezoneStr); + } + TAOS_CHECK_RETURN(cfgSetItem(pCfg, "timezone", tsTimezoneStr, pItem->stype, true)); + + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "locale"); + const char *locale = pItem->str; + + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "charset"); + const char *charset = pItem->str; + + int32_t code = taosSetSystemLocale(locale, charset); + if (TSDB_CODE_SUCCESS != code) { + uError("failed to set locale:%s, since: %s", locale, tstrerror(code)); + char curLocale[TD_LOCALE_LEN] = {0}; + char curCharset[TD_CHARSET_LEN] = {0}; + taosGetSystemLocale(curLocale, curCharset); + if (0 != strlen(curLocale) && 0 != strlen(curCharset)) { + uInfo("current locale: %s, charset: %s", curLocale, curCharset); + } + } + + osSetSystemLocale(locale, charset); + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "enableCoreFile"); tsEnableCoreFile = pItem->bval; taosSetCoreDump(tsEnableCoreFile); @@ -2012,6 +2039,7 @@ int32_t cfgDeserialize(SArray *array, char *buf, bool isGlobal) { pItem->str = taosStrdup(pJson->valuestring); if (pItem->str == NULL) { code = terrno; + goto _exit; } break; } @@ -2043,7 +2071,8 @@ int32_t readCfgFile(const char *path, bool isGlobal) { if (taosStatFile(filename, &fileSize, NULL, NULL) < 0) { if (terrno != ENOENT) { code = terrno; - uTrace("failed to stat file:%s , since %s", filename, tstrerror(code)); + uError("failed to stat file:%s , since %s", filename, tstrerror(code)); + TAOS_RETURN(TSDB_CODE_SUCCESS); } uInfo("config file:%s does not exist", filename); TAOS_RETURN(TSDB_CODE_SUCCESS); @@ -2407,9 +2436,6 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; - if (strcasecmp("charset", name) == 0 || strcasecmp("timezone", name) == 0) { - goto _out; - } cfgLock(pCfg); SConfigItem *pItem = cfgGetItem(pCfg, name); @@ -2501,15 +2527,18 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { case 'l': { if (strcasecmp("locale", name) == 0) { SConfigItem *pLocaleItem = cfgGetItem(pCfg, "locale"); - if (pLocaleItem == NULL) { - uError("failed to get locale from cfg"); + SConfigItem *pCharsetItem = cfgGetItem(pCfg, "charset"); + if (pLocaleItem == NULL || pCharsetItem == NULL) { + uError("failed to get locale or charset from cfg"); code = TSDB_CODE_CFG_NOT_FOUND; goto _out; } const char *locale = pLocaleItem->str; - TAOS_CHECK_GOTO(taosSetSystemLocale(locale), &lino, _out); - uInfo("locale set to '%s'", locale); + const char *charset = pCharsetItem->str; + TAOS_CHECK_GOTO(taosSetSystemLocale(locale, charset), &lino, _out); + osSetSystemLocale(locale, charset); + uInfo("locale set to '%s', charset set to '%s'", locale, charset); matched = true; } break; @@ -2584,7 +2613,13 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { break; } case 't': { - if (strcasecmp("tempDir", name) == 0) { + if (strcasecmp("timezone", name) == 0) { + TAOS_CHECK_GOTO(osSetTimezone(pItem->str), &lino, _out); + uInfo("%s set from %s to %s", name, tsTimezoneStr, pItem->str); + + TAOS_CHECK_GOTO(cfgSetItem(pCfg, "timezone", tsTimezoneStr, pItem->stype, false), &lino, _out); + matched = true; + } else if (strcasecmp("tempDir", name) == 0) { uInfo("%s set from %s to %s", name, tsTempDir, pItem->str); tstrncpy(tsTempDir, pItem->str, PATH_MAX); TAOS_CHECK_GOTO(taosExpandDir(tsTempDir, tsTempDir, PATH_MAX), &lino, _out); From ea767b28fe7ef9e7f6a843da30002f7df825c186 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 10 Dec 2024 15:12:01 +0800 Subject: [PATCH 77/99] Fix merge errors. --- source/common/src/tglobal.c | 55 +++++---------------- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- 2 files changed, 14 insertions(+), 43 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index bab37d72c6..e0274ec57b 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -335,6 +335,7 @@ bool tsExperimental = true; int32_t tsMaxTsmaNum = 3; int32_t tsMaxTsmaCalcDelay = 600; int64_t tsmaDataDeleteMark = 1000 * 60 * 60 * 24; // in ms, default to 1d +void *pTimezoneNameMap = NULL; #define TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, pName) \ if ((pItem = cfgGetItem(pCfg, pName)) == NULL) { \ @@ -1418,38 +1419,13 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { static int32_t taosSetSystemCfg(SConfig *pCfg) { SConfigItem *pItem = NULL; - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "timezone"); - if (0 == strlen(pItem->str)) { - uError("timezone is not set"); - } else { - TAOS_CHECK_RETURN(osSetTimezone(pItem->str)); - uDebug("timezone format changed from %s to %s", pItem->str, tsTimezoneStr); - } - TAOS_CHECK_RETURN(cfgSetItem(pCfg, "timezone", tsTimezoneStr, pItem->stype, true)); - - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "locale"); - const char *locale = pItem->str; - - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "charset"); - const char *charset = pItem->str; - - int32_t code = taosSetSystemLocale(locale, charset); - if (TSDB_CODE_SUCCESS != code) { - uError("failed to set locale:%s, since: %s", locale, tstrerror(code)); - char curLocale[TD_LOCALE_LEN] = {0}; - char curCharset[TD_CHARSET_LEN] = {0}; - taosGetSystemLocale(curLocale, curCharset); - if (0 != strlen(curLocale) && 0 != strlen(curCharset)) { - uInfo("current locale: %s, charset: %s", curLocale, curCharset); - } - } - - osSetSystemLocale(locale, charset); - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "enableCoreFile"); tsEnableCoreFile = pItem->bval; taosSetCoreDump(tsEnableCoreFile); + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "assert"); + tsAssert = pItem->bval; + // todo tsVersion = 30000000; @@ -2436,6 +2412,10 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; + if (strcasecmp("charset", name) == 0 || strcasecmp("timezone", name) == 0) { + goto _out; + } + cfgLock(pCfg); SConfigItem *pItem = cfgGetItem(pCfg, name); @@ -2527,18 +2507,15 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { case 'l': { if (strcasecmp("locale", name) == 0) { SConfigItem *pLocaleItem = cfgGetItem(pCfg, "locale"); - SConfigItem *pCharsetItem = cfgGetItem(pCfg, "charset"); - if (pLocaleItem == NULL || pCharsetItem == NULL) { - uError("failed to get locale or charset from cfg"); + if (pLocaleItem == NULL) { + uError("failed to get locale from cfg"); code = TSDB_CODE_CFG_NOT_FOUND; goto _out; } const char *locale = pLocaleItem->str; - const char *charset = pCharsetItem->str; - TAOS_CHECK_GOTO(taosSetSystemLocale(locale, charset), &lino, _out); - osSetSystemLocale(locale, charset); - uInfo("locale set to '%s', charset set to '%s'", locale, charset); + TAOS_CHECK_GOTO(taosSetSystemLocale(locale), &lino, _out); + uInfo("locale set to '%s'", locale); matched = true; } break; @@ -2613,13 +2590,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { break; } case 't': { - if (strcasecmp("timezone", name) == 0) { - TAOS_CHECK_GOTO(osSetTimezone(pItem->str), &lino, _out); - uInfo("%s set from %s to %s", name, tsTimezoneStr, pItem->str); - - TAOS_CHECK_GOTO(cfgSetItem(pCfg, "timezone", tsTimezoneStr, pItem->stype, false), &lino, _out); - matched = true; - } else if (strcasecmp("tempDir", name) == 0) { + if (strcasecmp("tempDir", name) == 0) { uInfo("%s set from %s to %s", name, tsTempDir, pItem->str); tstrncpy(tsTempDir, pItem->str, PATH_MAX); TAOS_CHECK_GOTO(taosExpandDir(tsTempDir, tsTempDir, PATH_MAX), &lino, _out); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 82ec6fc131..906cc30946 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -478,7 +478,7 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { SConfig *pCfg = taosGetCfg(); SConfigItem *pItem = NULL; - code = cfgGetAndSetItem(pCfg, &pItem, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_CMD, true); + code = cfgGetAndSetItem(pCfg, &pItem, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_SERVER_CMD, true); if (code != 0) { if (strncasecmp(cfgReq.config, "resetlog", strlen("resetlog")) == 0) { TAOS_CHECK_RETURN(taosCfgDynamicOptions(pCfg, cfgReq.config, true)); From b34dd6f3472a6c1774861fb358df62a368f65eea Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 10 Dec 2024 18:56:59 +0800 Subject: [PATCH 78/99] Fix some review errors. --- source/common/src/tglobal.c | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index e0274ec57b..8defbbc546 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -21,9 +21,11 @@ #include "tconfig.h" #include "tglobal.h" #include "tgrant.h" +#include "tjson.h" #include "tlog.h" #include "tmisce.h" #include "tunit.h" + #include "tutil.h" #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) @@ -2042,27 +2044,32 @@ int32_t readCfgFile(const char *path, bool isGlobal) { } uInfo("start to read config file:%s", filename); - int64_t fileSize = 0; - char *buf = NULL; - if (taosStatFile(filename, &fileSize, NULL, NULL) < 0) { - if (terrno != ENOENT) { - code = terrno; - uError("failed to stat file:%s , since %s", filename, tstrerror(code)); - TAOS_RETURN(TSDB_CODE_SUCCESS); - } + if (!taosCheckExistFile(filename)) { uInfo("config file:%s does not exist", filename); TAOS_RETURN(TSDB_CODE_SUCCESS); } + int64_t fileSize = 0; + char *buf = NULL; + if (taosStatFile(filename, &fileSize, NULL, NULL) < 0) { + code = terrno; + uError("failed to stat file:%s , since %s", filename, tstrerror(code)); + TAOS_RETURN(code); + } TdFilePtr pFile = taosOpenFile(filename, TD_FILE_READ); if (pFile == NULL) { - uError("failed to open file:%s , since %s", filename, tstrerror(code)); code = terrno; + uError("failed to open file:%s , since %s", filename, tstrerror(code)); goto _exit; } buf = (char *)taosMemoryMalloc(fileSize + 1); - if (taosReadFile(pFile, buf, fileSize) != fileSize) { - uError("failed to read file:%s , config since %s", filename, tstrerror(code)); + if (buf == NULL) { code = terrno; + uError("failed to malloc memory for file:%s , since %s", filename, tstrerror(code)); + goto _exit; + } + if (taosReadFile(pFile, buf, fileSize) != fileSize) { + code = terrno; + uError("failed to read file:%s , config since %s", filename, tstrerror(code)); goto _exit; } buf[fileSize] = 0; @@ -2761,7 +2768,6 @@ int32_t globalConfigSerialize(int32_t version, SArray *array, char **serialized) cJSON *json = cJSON_CreateObject(); if (json == NULL) goto _exit; if (cJSON_AddNumberToObject(json, "version", version) == NULL) goto _exit; - if (json == NULL) goto _exit; int sz = taosArrayGetSize(array); cJSON *cField = cJSON_CreateObject(); @@ -2802,7 +2808,7 @@ int32_t globalConfigSerialize(int32_t version, SArray *array, char **serialized) } } } - char *pSerialized = cJSON_Print(json); + char *pSerialized = tjsonToString(json); _exit: if (terrno != TSDB_CODE_SUCCESS) { uError("failed to serialize global config since %s", tstrerror(terrno)); @@ -2833,6 +2839,7 @@ int32_t localConfigSerialize(SArray *array, char **serialized) { cJSON *dataDirs = cJSON_CreateArray(); if (!cJSON_AddItemToObject(cField, item->name, dataDirs)) { uError("failed to serialize global config since %s", tstrerror(terrno)); + goto _exit; } for (int j = 0; j < sz; j++) { SDiskCfg *disk = (SDiskCfg *)taosArrayGet(item->array, j); @@ -2840,6 +2847,7 @@ int32_t localConfigSerialize(SArray *array, char **serialized) { if (dataDir == NULL) goto _exit; if (!cJSON_AddItemToArray(dataDirs, dataDir)) { uError("failed to serialize global config since %s", tstrerror(terrno)); + goto _exit; } if (cJSON_AddStringToObject(dataDir, "dir", disk->dir) == NULL) goto _exit; if (cJSON_AddNumberToObject(dataDir, "level", disk->level) == NULL) goto _exit; @@ -2885,7 +2893,7 @@ int32_t localConfigSerialize(SArray *array, char **serialized) { } } } - char *pSerialized = cJSON_Print(json); + char *pSerialized = tjsonToString(json); _exit: if (terrno != TSDB_CODE_SUCCESS) { uError("failed to serialize local config since %s", tstrerror(terrno)); From 28ea9b9f601d9a56e3d1c4e90d83418e5ea58082 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 10 Dec 2024 20:13:53 +0800 Subject: [PATCH 79/99] Fix review errors. --- source/common/src/tglobal.c | 15 ++++----- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 11 ++++++- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 1 - source/dnode/mnode/impl/src/mndConfig.c | 24 ++++++++++++--- source/util/src/tconfig.c | 34 +++++++++++++-------- 5 files changed, 58 insertions(+), 27 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 8defbbc546..d6add7f3d9 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1425,9 +1425,6 @@ static int32_t taosSetSystemCfg(SConfig *pCfg) { tsEnableCoreFile = pItem->bval; taosSetCoreDump(tsEnableCoreFile); - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "assert"); - tsAssert = pItem->bval; - // todo tsVersion = 30000000; @@ -1981,7 +1978,11 @@ int32_t cfgDeserialize(SArray *array, char *buf, bool isGlobal) { int64_t actDiskID = 0; int64_t expDiskID = taosStr2Int64(cJSON_GetStringValue(filed), NULL, 10); - if (!taosCheckFileDiskID(dir, &actDiskID, expDiskID)) { + if ((code = taosGetFileDiskID(dir, &actDiskID)) != 0) { + uError("failed to get disk id for dir:%s, since %s", dir, tstrerror(code)); + goto _exit; + } + if (actDiskID != expDiskID) { uError("failed to check disk id for dir:%s, actDiskID%" PRId64 ", expDiskID%" PRId64, dir, actDiskID, expDiskID); code = TSDB_CODE_INVALID_DISK_ID; @@ -2006,7 +2007,7 @@ int32_t cfgDeserialize(SArray *array, char *buf, bool isGlobal) { break; case CFG_DTYPE_FLOAT: case CFG_DTYPE_DOUBLE: - pItem->fval = atoll(cJSON_GetStringValue(pJson)); + pItem->fval = taosStr2Float(cJSON_GetStringValue(pJson), NULL); break; case CFG_DTYPE_STRING: case CFG_DTYPE_DIR: @@ -2959,7 +2960,7 @@ int32_t taosPersistLocalConfig(const char *path) { taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pConfigFile == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(terrno); uError("failed to open file:%s since %s", filename, tstrerror(code)); TAOS_RETURN(code); } @@ -2968,7 +2969,7 @@ int32_t taosPersistLocalConfig(const char *path) { TAOS_CHECK_GOTO(localConfigSerialize(taosGetLocalCfg(tsCfg), &serialized), &lino, _exit); if (taosWriteFile(pConfigFile, serialized, strlen(serialized)) < 0) { lino = __LINE__; - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(terrno); uError("failed to write file:%s since %s", filename, tstrerror(code)); goto _exit; } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 906cc30946..f9a0ef8925 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -198,7 +198,8 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { req.clusterCfg.monitorParas.tsSlowLogThreshold = tsSlowLogThreshold; tstrncpy(req.clusterCfg.monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb, TSDB_DB_NAME_LEN); char timestr[32] = "1970-01-01 00:00:00.00"; - if (taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, NULL) != 0) { + if (taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, NULL) != + 0) { dError("failed to parse time since %s", tstrerror(code)); } memcpy(req.clusterCfg.timezone, tsTimezoneStr, TD_TIMEZONE_LEN); @@ -333,6 +334,10 @@ static void dmProcessConfigRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { } if (needUpdate) { code = cfgUpdateFromArray(tsCfg, configRsp.array); + if (code != TSDB_CODE_SUCCESS) { + dError("failed to update config since %s", tstrerror(code)); + goto _exit; + } code = setAllConfigs(tsCfg); if (code != TSDB_CODE_SUCCESS) { dError("failed to set all configs since %s", tstrerror(code)); @@ -369,6 +374,10 @@ void dmSendConfigReq(SDnodeMgmt *pMgmt) { } void *pHead = rpcMallocCont(contLen); + if (pHead == NULL) { + dError("failed to malloc cont since %s", tstrerror(contLen)); + return; + } contLen = tSerializeSConfigReq(pHead, contLen, &req); if (contLen < 0) { rpcFreeCont(pHead); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index b481a3fc5e..fb810ef0c4 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -51,7 +51,6 @@ static void *dmConfigThreadFp(void *param) { SDnodeMgmt *pMgmt = param; int64_t lastTime = taosGetTimestampMs(); setThreadName("dnode-config"); - int32_t upTimeCount = 0; while (1) { taosMsleep(200); if (pMgmt->pData->dropped || pMgmt->pData->stopped || tsConfigInited) break; diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index 1155495fd4..0143a5d367 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -244,6 +244,10 @@ static int32_t mndProcessConfigReq(SRpcMsg *pReq) { } array = taosArrayInit(16, sizeof(SConfigItem)); + if (array == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } SConfigRsp configRsp = {0}; configRsp.forceReadConfig = configReq.forceReadConfig; @@ -291,6 +295,7 @@ _OVER: if (code != 0) { mError("failed to process config req, since %s", tstrerror(code)); } + sdbRelease(pMnode->pSdb, vObj); cfgArrayCleanUp(array); mndReleaseDnode(pMnode, pDnode); return TSDB_CODE_SUCCESS; @@ -310,7 +315,7 @@ int32_t mndInitWriteCfg(SMnode *pMnode) { // encode mnd config version SConfigObj *versionObj = mndInitConfigVersion(); if ((code = mndSetCreateConfigCommitLogs(pTrans, versionObj)) != 0) { - mError("failed to init mnd config version, since %s", terrstr()); + mError("failed to init mnd config version, since %s", tstrerror(code)); taosMemoryFree(versionObj->str); taosMemoryFree(versionObj); goto _OVER; @@ -327,6 +332,8 @@ int32_t mndInitWriteCfg(SMnode *pMnode) { } if ((code = mndSetCreateConfigCommitLogs(pTrans, obj)) != 0) { mError("failed to init mnd config:%s, since %s", item->name, tstrerror(code)); + taosMemoryFree(obj); + goto _OVER; } taosMemoryFree(obj); } @@ -347,7 +354,7 @@ int32_t mndInitReadCfg(SMnode *pMnode) { if (obj == NULL) { code = mndInitWriteCfg(pMnode); if (code != 0) { - mError("failed to init write cfg, since %s", terrstr()); + mError("failed to init write cfg, since %s", tstrerror(code)); } mInfo("failed to acquire mnd config version, try to rebuild it , since %s", terrstr()); goto _OVER; @@ -426,17 +433,17 @@ static int32_t mndMCfg2DCfg(SMCfgDnodeReq *pMCfgReq, SDCfgDnodeReq *pDCfgReq) { } size_t optLen = p - pMCfgReq->config; - tstrncpy(pDCfgReq->config, pMCfgReq->config, optLen + 1); + strncpy(pDCfgReq->config, pMCfgReq->config, optLen); pDCfgReq->config[optLen] = 0; if (' ' == pMCfgReq->config[optLen]) { // 'key value' if (strlen(pMCfgReq->value) != 0) goto _err; - tstrncpy(pDCfgReq->value, p + 1, strlen(p + 1)); + (void)strcpy(pDCfgReq->value, p + 1); } else { // 'key' 'value' if (strlen(pMCfgReq->value) == 0) goto _err; - tstrncpy(pDCfgReq->value, pMCfgReq->value, strlen(pMCfgReq->value)); + (void)strcpy(pDCfgReq->value, pMCfgReq->value); } TAOS_RETURN(code); @@ -659,6 +666,7 @@ static int32_t initConfigArrayFromSdb(SMnode *pMnode, SArray *array) { if (item.name == NULL) { code = terrno; sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, obj); goto _exit; } switch (obj->dtype) { @@ -685,6 +693,7 @@ static int32_t initConfigArrayFromSdb(SMnode *pMnode, SArray *array) { item.str = taosStrdup(obj->str); if (item.str == NULL) { sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, obj); code = terrno; goto _exit; } @@ -692,6 +701,7 @@ static int32_t initConfigArrayFromSdb(SMnode *pMnode, SArray *array) { } if (taosArrayPush(array, &item) == NULL) { sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, obj); code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; break; @@ -731,6 +741,10 @@ SArray *initVariablesFromItems(SArray *pItems) { int32_t sz = taosArrayGetSize(pItems); SArray *pInfos = taosArrayInit(sz, sizeof(SVariablesInfo)); + if (pInfos == NULL) { + mError("failed to init array while init variables from items, since %s", tstrerror(terrno)); + return NULL; + } for (int32_t i = 0; i < sz; ++i) { SConfigItem *pItem = taosArrayGet(pItems, i); SVariablesInfo info = {0}; diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index a6b2fc6a05..f18fdfe970 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -17,6 +17,7 @@ #include "cJSON.h" #include "taoserror.h" #include "tconfig.h" +#include "tconv.h" #include "tenv.h" #include "tglobal.h" #include "tgrant.h" @@ -24,7 +25,6 @@ #include "tlog.h" #include "tunit.h" #include "tutil.h" -#include "tconv.h" #define CFG_NAME_PRINT_LEN 32 #define CFG_SRC_PRINT_LEN 12 @@ -129,6 +129,10 @@ int32_t cfgUpdateFromArray(SConfig *pCfg, SArray *pArgs) { case CFG_DTYPE_TIMEZONE: taosMemoryFree(pItemOld->str); pItemOld->str = taosStrdup(pItemNew->str); + if (pItemOld->str == NULL) { + (void)taosThreadMutexUnlock(&pCfg->lock); + TAOS_RETURN(terrno); + } break; default: break; @@ -307,7 +311,7 @@ static int32_t doSetConf(SConfigItem *pItem, const char *value, ECfgSrcType styp } static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType stype) { - if (stype == CFG_STYPE_ALTER_SERVER_CMD || (pItem->dynScope & CFG_DYN_CLIENT) == 0){ + if (stype == CFG_STYPE_ALTER_SERVER_CMD || (pItem->dynScope & CFG_DYN_CLIENT) == 0) { uError("failed to config timezone, not support"); TAOS_RETURN(TSDB_CODE_INVALID_CFG); } @@ -325,7 +329,7 @@ static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType } static int32_t cfgSetCharset(SConfigItem *pItem, const char *value, ECfgSrcType stype) { - if (stype == CFG_STYPE_ALTER_SERVER_CMD || stype == CFG_STYPE_ALTER_CLIENT_CMD){ + if (stype == CFG_STYPE_ALTER_SERVER_CMD || stype == CFG_STYPE_ALTER_CLIENT_CMD) { uError("failed to config charset, not support"); TAOS_RETURN(TSDB_CODE_INVALID_CFG); } @@ -351,7 +355,7 @@ static int32_t cfgSetCharset(SConfigItem *pItem, const char *value, ECfgSrcType } static int32_t cfgSetLocale(SConfigItem *pItem, const char *value, ECfgSrcType stype) { - if (stype == CFG_STYPE_ALTER_SERVER_CMD || (pItem->dynScope & CFG_DYN_CLIENT) == 0){ + if (stype == CFG_STYPE_ALTER_SERVER_CMD || (pItem->dynScope & CFG_DYN_CLIENT) == 0) { uError("failed to config locale, not support"); TAOS_RETURN(TSDB_CODE_INVALID_CFG); } @@ -589,7 +593,11 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p cfgUnLock(pCfg); TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); } - TAOS_CHECK_RETURN(checkItemDyn(pItem, isServer)); + int32_t code = checkItemDyn(pItem, isServer); + if (code != TSDB_CODE_SUCCESS) { + cfgUnLock(pCfg); + TAOS_RETURN(code); + } if ((!isUpdateAll) && (pItem->category == CFG_CATEGORY_GLOBAL)) { uError("failed to config:%s, not support update global config on only one dnode", name); cfgUnLock(pCfg); @@ -872,14 +880,14 @@ int32_t cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t case CFG_DTYPE_DOUBLE: len = tsnprintf(buf, bufSize, "%f", pItem->fval); break; - case CFG_DTYPE_TIMEZONE:{ -// char str1[TD_TIMEZONE_LEN] = {0}; -// time_t tx1 = taosGetTimestampSec(); -// if (taosFormatTimezoneStr(tx1, buf, NULL, str1) != 0) { -// tstrncpy(str1, "tz error", sizeof(str1)); -// } -// len = tsnprintf(buf, bufSize, "%s", str1); -// break; + case CFG_DTYPE_TIMEZONE: { + // char str1[TD_TIMEZONE_LEN] = {0}; + // time_t tx1 = taosGetTimestampSec(); + // if (taosFormatTimezoneStr(tx1, buf, NULL, str1) != 0) { + // tstrncpy(str1, "tz error", sizeof(str1)); + // } + // len = tsnprintf(buf, bufSize, "%s", str1); + // break; } case CFG_DTYPE_STRING: case CFG_DTYPE_DIR: From 99f4aadef12c4e07562d2386946d06b0089dffd2 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 11 Dec 2024 09:35:35 +0800 Subject: [PATCH 80/99] Fix review errors. --- source/dnode/mnode/impl/src/mndConfig.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index 0143a5d367..d9ccd2f5af 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -836,6 +836,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { } if ((rspLen = tSerializeSShowVariablesRsp(pRsp, rspLen, &rsp)) <= 0) { + rpcFreeCont(pRsp); code = rspLen; goto _OVER; } From b3f1a0ca591492c2482f78f1ec60960941010173 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 11 Dec 2024 09:36:23 +0800 Subject: [PATCH 81/99] Fix review errors. --- source/os/src/osFile.c | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 27450b810f..8a2606c4c2 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -323,26 +323,6 @@ int32_t taosGetFileDiskID(const char *path, int64_t *diskid) { return 0; } -bool taosCheckFileDiskID(const char *path, int64_t *actDiskID, int64_t expDiskID) { - OS_PARAM_CHECK(path); -#ifdef WINDOWS - struct _stati64 fileStat; - int32_t code = _stati64(path, &fileStat); -#else - struct stat fileStat; - int32_t code = stat(path, &fileStat); -#endif - if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; - } - - if (actDiskID != NULL) { - *actDiskID = fileStat.st_dev; - } - return fileStat.st_dev == expDiskID; -} - int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { #ifdef WINDOWS if (pFile == NULL || pFile->hFile == NULL) { From 40478a9bfaeb653ee1c1dbeb0631edc1d1101f64 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 11 Dec 2024 14:00:38 +0800 Subject: [PATCH 82/99] Fix ci problems. --- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index fb810ef0c4..6e30c8a1bf 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -368,7 +368,6 @@ void dmStopStatusThread(SDnodeMgmt *pMgmt) { void dmStopConfigThread(SDnodeMgmt *pMgmt) { if (taosCheckPthreadValid(pMgmt->configThread)) { - (void)taosThreadJoin(pMgmt->configThread, NULL); taosThreadClear(&pMgmt->configThread); } } From 2abb383ec0125012cce0504e1d4e4fd209a1b43f Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 11 Dec 2024 15:11:08 +0800 Subject: [PATCH 83/99] Fix ci problems. --- source/dnode/mgmt/mgmt_dnode/inc/dmInt.h | 1 - source/dnode/mgmt/mgmt_dnode/src/dmInt.c | 1 - source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 6 ------ 3 files changed, 8 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index bfe4cd165e..2108a097ee 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -69,7 +69,6 @@ int32_t dmStartStatusThread(SDnodeMgmt *pMgmt); int32_t dmStartConfigThread(SDnodeMgmt *pMgmt); int32_t dmStartStatusInfoThread(SDnodeMgmt *pMgmt); void dmStopStatusThread(SDnodeMgmt *pMgmt); -void dmStopConfigThread(SDnodeMgmt *pMgmt); void dmStopStatusInfoThread(SDnodeMgmt *pMgmt); int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt); void dmStopNotifyThread(SDnodeMgmt *pMgmt); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index b3b1df314a..b58c1a216d 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -52,7 +52,6 @@ static void dmStopMgmt(SDnodeMgmt *pMgmt) { dmStopMonitorThread(pMgmt); dmStopAuditThread(pMgmt); dmStopStatusThread(pMgmt); - dmStopConfigThread(pMgmt); dmStopStatusInfoThread(pMgmt); #if defined(TD_ENTERPRISE) dmStopNotifyThread(pMgmt); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 6e30c8a1bf..6372374526 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -366,12 +366,6 @@ void dmStopStatusThread(SDnodeMgmt *pMgmt) { } } -void dmStopConfigThread(SDnodeMgmt *pMgmt) { - if (taosCheckPthreadValid(pMgmt->configThread)) { - taosThreadClear(&pMgmt->configThread); - } -} - void dmStopStatusInfoThread(SDnodeMgmt *pMgmt) { if (taosCheckPthreadValid(pMgmt->statusInfoThread)) { (void)taosThreadJoin(pMgmt->statusInfoThread, NULL); From c50781d08971db204e6ffc8e0993e255e217d05f Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 11 Dec 2024 15:46:44 +0800 Subject: [PATCH 84/99] Send alter msg sync. --- include/util/tconfig.h | 4 +- source/dnode/mnode/impl/src/mndConfig.c | 67 +++++++++++++++++-------- source/libs/command/src/command.c | 8 +-- source/libs/transport/src/tmsgcb.c | 1 + source/util/src/tconfig.c | 5 +- tests/army/alter/test_alter_config.py | 5 -- 6 files changed, 59 insertions(+), 31 deletions(-) diff --git a/include/util/tconfig.h b/include/util/tconfig.h index dfee25f680..cc4c605ad4 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -54,6 +54,7 @@ typedef enum { typedef enum { CFG_SCOPE_SERVER, CFG_SCOPE_CLIENT, CFG_SCOPE_BOTH } ECfgScopeType; typedef enum { CFG_CATEGORY_GLOBAL, CFG_CATEGORY_LOCAL } ECfgCategoryType; +typedef enum { CFG_ALTER_LOCAL, CFG_ALTER_DNODE, CFG_ALTER_ALL_DNODES } CfgAlterType; typedef enum { CFG_DYN_NONE = 0, @@ -123,7 +124,8 @@ SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName); int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype, bool lock); int32_t cfgGetAndSetItem(SConfig *pCfg, SConfigItem **ppItem, const char *name, const char *value, ECfgSrcType stype, bool lock); -int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer, bool isUpdateAll); +int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer, + CfgAlterType alterType); int32_t cfgCreateIter(SConfig *pConf, SConfigIter **ppIter); SConfigItem *cfgNextIter(SConfigIter *pIter); diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index d9ccd2f5af..cd9a8f7405 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -22,13 +22,9 @@ #include "mndUser.h" #include "tutil.h" -#define CFG_VER_NUMBER 1 -#define CFG_RESERVE_SIZE 63 - -enum CfgAlterType { - CFG_ALTER_DNODE, - CFG_ALTER_ALL_DNODES, -}; +#define CFG_VER_NUMBER 1 +#define CFG_RESERVE_SIZE 63 +#define CFG_ALTER_TIMEOUT 3 * 1000 static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pInMCfgReq, int32_t optLen, int32_t *pOutValue); static int32_t cfgUpdateItem(SConfigItem *pItem, SConfigObj *obj); @@ -458,29 +454,59 @@ static int32_t mndSendCfgDnodeReq(SMnode *pMnode, int32_t dnodeId, SDCfgDnodeReq int32_t code = -1; SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; + + int64_t curMs = taosGetTimestampMs(); + while (1) { SDnodeObj *pDnode = NULL; pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); if (pIter == NULL) break; if (pDnode->id == dnodeId || dnodeId == -1 || dnodeId == 0) { + bool online = mndIsDnodeOnline(pDnode, curMs); + if (!online) { + mWarn("dnode:%d, is offline, skip to send config req", pDnode->id); + continue; + } SEpSet epSet = mndGetDnodeEpset(pDnode); int32_t bufLen = tSerializeSDCfgDnodeReq(NULL, 0, pDcfgReq); void *pBuf = rpcMallocCont(bufLen); - if (pBuf != NULL) { - if ((bufLen = tSerializeSDCfgDnodeReq(pBuf, bufLen, pDcfgReq)) <= 0) { - sdbCancelFetch(pMnode->pSdb, pIter); - sdbRelease(pMnode->pSdb, pDnode); - code = bufLen; - return code; - } - mInfo("dnode:%d, send config req to dnode, config:%s value:%s", dnodeId, pDcfgReq->config, pDcfgReq->value); - SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen}; - code = tmsgSendReq(&epSet, &rpcMsg); + if (pBuf == NULL) { + sdbCancelFetch(pMnode->pSdb, pIter); + sdbRelease(pMnode->pSdb, pDnode); + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + + if ((bufLen = tSerializeSDCfgDnodeReq(pBuf, bufLen, pDcfgReq)) <= 0) { + sdbCancelFetch(pMnode->pSdb, pIter); + sdbRelease(pMnode->pSdb, pDnode); + code = bufLen; + rpcFreeCont(pBuf); + return code; + } + + mInfo("dnode:%d, send config req to dnode, config:%s value:%s", pDnode->id, pDcfgReq->config, pDcfgReq->value); + SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen}; + SRpcMsg rpcRsp = {0}; + + code = rpcSendRecvWithTimeout(pMnode->msgCb.statusRpc, &epSet, &rpcMsg, &rpcRsp, NULL, CFG_ALTER_TIMEOUT); + if (code != 0) { + mError("failed to send config req to dnode:%d, since %s", pDnode->id, tstrerror(code)); + sdbCancelFetch(pMnode->pSdb, pIter); + sdbRelease(pMnode->pSdb, pDnode); + return code; + } + + code = rpcRsp.code; + if (code != 0) { + mError("failed to alter config %s,on dnode:%d, since %s", pDcfgReq->config, pDnode->id, tstrerror(code)); + sdbCancelFetch(pMnode->pSdb, pIter); + sdbRelease(pMnode->pSdb, pDnode); + return code; } } - sdbRelease(pSdb, pDnode); } @@ -541,8 +567,8 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { updateIpWhiteList = 1; } - bool isUpdateAll = (cfgReq.dnodeId == 0 || cfgReq.dnodeId == -1) ? true : false; - TAOS_CHECK_GOTO(cfgCheckRangeForDynUpdate(taosGetCfg(), dcfgReq.config, dcfgReq.value, true, isUpdateAll), &lino, + CfgAlterType alterType = (cfgReq.dnodeId == 0 || cfgReq.dnodeId == -1) ? CFG_ALTER_ALL_DNODES : CFG_ALTER_DNODE; + TAOS_CHECK_GOTO(cfgCheckRangeForDynUpdate(taosGetCfg(), dcfgReq.config, dcfgReq.value, true, alterType), &lino, _err_out); } SConfigItem *pItem = cfgGetItem(taosGetCfg(), dcfgReq.config); @@ -658,6 +684,7 @@ static int32_t initConfigArrayFromSdb(SMnode *pMnode, SArray *array) { goto _exit; } if (strcasecmp(obj->name, "tsmmConfigVersion") == 0) { + sdbRelease(pSdb, obj); continue; } SConfigItem item = {0}; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index f16c28f4ef..d0656026f1 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -726,7 +726,8 @@ static void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STab } } -static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* pDbCfg, char* tbName, STableCfg* pCfg, void* charsetCxt) { +static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* pDbCfg, char* tbName, STableCfg* pCfg, + void* charsetCxt) { int32_t code = TSDB_CODE_SUCCESS; QRY_ERR_RET(blockDataEnsureCapacity(pBlock, 1)); pBlock->info.rows = 1; @@ -905,7 +906,7 @@ static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { goto _return; } - if (cfgCheckRangeForDynUpdate(tsCfg, pStmt->config, pStmt->value, false, false)) { + if (cfgCheckRangeForDynUpdate(tsCfg, pStmt->config, pStmt->value, false, CFG_ALTER_LOCAL)) { return terrno; } @@ -1064,7 +1065,8 @@ static int32_t execShowCreateView(SShowCreateViewStmt* pStmt, SRetrieveTableRsp* return code; } -int32_t qExecCommand(int64_t* pConnId, bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp, int8_t biMode, void* charsetCxt) { +int32_t qExecCommand(int64_t* pConnId, bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp, int8_t biMode, + void* charsetCxt) { switch (nodeType(pStmt)) { case QUERY_NODE_DESCRIBE_STMT: return execDescribe(sysInfoUser, pStmt, pRsp, biMode); diff --git a/source/libs/transport/src/tmsgcb.c b/source/libs/transport/src/tmsgcb.c index 4c969003a9..fab80bde9c 100644 --- a/source/libs/transport/src/tmsgcb.c +++ b/source/libs/transport/src/tmsgcb.c @@ -48,6 +48,7 @@ int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg) { } return code; } + int32_t tmsgSendSyncReq(const SEpSet* epSet, SRpcMsg* pMsg) { int32_t code = (*defaultMsgCb.sendSyncReqFp)(epSet, pMsg); if (code != 0) { diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index f18fdfe970..75b3ce9b58 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -585,7 +585,8 @@ int32_t checkItemDyn(SConfigItem *pItem, bool isServer) { return TSDB_CODE_SUCCESS; } -int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer, bool isUpdateAll) { +int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer, + CfgAlterType alterType) { cfgLock(pCfg); SConfigItem *pItem = cfgGetItem(pCfg, name); @@ -598,7 +599,7 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p cfgUnLock(pCfg); TAOS_RETURN(code); } - if ((!isUpdateAll) && (pItem->category == CFG_CATEGORY_GLOBAL)) { + if ((pItem->category == CFG_CATEGORY_GLOBAL) && alterType == CFG_ALTER_DNODE) { uError("failed to config:%s, not support update global config on only one dnode", name); cfgUnLock(pCfg); TAOS_RETURN(TSDB_CODE_INVALID_CFG); diff --git a/tests/army/alter/test_alter_config.py b/tests/army/alter/test_alter_config.py index 52072dae5c..1b59293187 100644 --- a/tests/army/alter/test_alter_config.py +++ b/tests/army/alter/test_alter_config.py @@ -315,11 +315,6 @@ class TDTestCase: "value": 500, "category": "global" }, - { - "name": "syncElectInterval", - "value": 50000, - "category": "global" - }, { "name": "syncHeartbeatInterval", "value": 3000, From 94c91085d61e36484a9c2f558b1f0889d52e2843 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 11 Dec 2024 16:40:23 +0800 Subject: [PATCH 85/99] Add file version in json file. --- include/common/tglobal.h | 3 +++ source/common/src/tglobal.c | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index dbce6826a3..cb27e76b35 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -31,6 +31,9 @@ extern "C" { #define SLOW_LOG_TYPE_OTHERS 0x4 #define SLOW_LOG_TYPE_ALL 0x7 +#define GLOBAL_CONFIG_FILE_VERSION 1 +#define LOCAL_CONFIG_FILE_VERSION 1 + typedef enum { DND_CA_SM4 = 1, } EEncryptAlgor; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index d6add7f3d9..98e19b5195 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -2768,6 +2768,7 @@ int32_t globalConfigSerialize(int32_t version, SArray *array, char **serialized) char buf[30]; cJSON *json = cJSON_CreateObject(); if (json == NULL) goto _exit; + if (cJSON_AddNumberToObject(json, "file_version", GLOBAL_CONFIG_FILE_VERSION) == NULL) goto _exit; if (cJSON_AddNumberToObject(json, "version", version) == NULL) goto _exit; int sz = taosArrayGetSize(array); @@ -2828,7 +2829,7 @@ int32_t localConfigSerialize(SArray *array, char **serialized) { cJSON *cField = cJSON_CreateObject(); if (cField == NULL) goto _exit; - + if (cJSON_AddNumberToObject(json, "file_version", LOCAL_CONFIG_FILE_VERSION) == NULL) goto _exit; if (!cJSON_AddItemToObject(json, "configs", cField)) goto _exit; // cjson only support int32_t or double From 391efd6bb948950c446bbe5b50c1087057e5a7a9 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 11 Dec 2024 18:36:07 +0800 Subject: [PATCH 86/99] Fix ci errors. --- source/dnode/mnode/impl/src/mndConfig.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index cd9a8f7405..f7d0cd9d1b 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -535,10 +535,10 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { SDCfgDnodeReq dcfgReq = {0}; if (strcasecmp(cfgReq.config, "resetlog") == 0) { - tstrncpy(dcfgReq.config, "resetlog", 8); + tstrncpy(dcfgReq.config, "resetlog", 9); goto _send_req; #ifdef TD_ENTERPRISE - } else if (strncasecmp(cfgReq.config, "s3blocksize", 11) == 0) { + } else if (strncasecmp(cfgReq.config, "s3blocksize", 12) == 0) { int32_t optLen = strlen("s3blocksize"); int32_t flag = -1; int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); @@ -553,7 +553,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { goto _err_out; } - tstrncpy(dcfgReq.config, "s3blocksize", 11); + tstrncpy(dcfgReq.config, "s3blocksize", 12); snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); #endif } else { From 9f692705c7412bce31bad4c5a9096baf8c8e9285 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 12 Dec 2024 09:42:56 +0800 Subject: [PATCH 87/99] Fix ci mem leak. --- source/dnode/mnode/impl/src/mndConfig.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index f7d0cd9d1b..7b32b8348d 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -506,6 +506,7 @@ static int32_t mndSendCfgDnodeReq(SMnode *pMnode, int32_t dnodeId, SDCfgDnodeReq sdbRelease(pMnode->pSdb, pDnode); return code; } + rpcFreeCont(rpcRsp.pCont); } sdbRelease(pSdb, pDnode); } From e2e5f0de4a646b43e0ba9ef4621c570a72d611fb Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 12 Dec 2024 11:17:21 +0800 Subject: [PATCH 88/99] Add func rebuilt cfg in sdb while add new global cfg. --- source/dnode/mnode/impl/src/mndConfig.c | 81 ++++++++++++++++++++----- 1 file changed, 67 insertions(+), 14 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index 7b32b8348d..a18e524581 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -33,9 +33,10 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq); static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessConfigReq(SRpcMsg *pReq); static int32_t mndInitWriteCfg(SMnode *pMnode); -static int32_t mndInitReadCfg(SMnode *pMnode); +static int32_t mndTryRebuildCfg(SMnode *pMnode); static int32_t initConfigArrayFromSdb(SMnode *pMnode, SArray *array); static void cfgArrayCleanUp(SArray *array); +static void cfgObjArrayCleanUp(SArray *array); static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue, ECfgDataType dtype, int32_t tsmmConfigVersion); @@ -218,7 +219,7 @@ static int32_t mndCfgActionUpdate(SSdb *pSdb, SConfigObj *pOld, SConfigObj *pNew static int32_t mndCfgActionDeploy(SMnode *pMnode) { return mndInitWriteCfg(pMnode); } -static int32_t mndCfgActionPrepare(SMnode *pMnode) { return mndInitReadCfg(pMnode); } +static int32_t mndCfgActionPrepare(SMnode *pMnode) { return mndTryRebuildCfg(pMnode); } static int32_t mndProcessConfigReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; @@ -343,20 +344,60 @@ _OVER: return code; } -int32_t mndInitReadCfg(SMnode *pMnode) { - int32_t code = 0; - int32_t sz = -1; - SConfigObj *obj = sdbAcquire(pMnode->pSdb, SDB_CFG, "tsmmConfigVersion"); - if (obj == NULL) { - code = mndInitWriteCfg(pMnode); - if (code != 0) { - mError("failed to init write cfg, since %s", tstrerror(code)); +int32_t mndTryRebuildCfg(SMnode *pMnode) { + int32_t code = 0; + int32_t sz = -1; + STrans *pTrans = NULL; + SAcctObj *vObj = NULL, *obj = NULL; + SArray *addArray = NULL; + vObj = sdbAcquire(pMnode->pSdb, SDB_CFG, "tsmmConfigVersion"); + if (vObj == NULL) { + if ((code = mndInitWriteCfg(pMnode)) < 0) goto _exit; + mInfo("failed to acquire mnd config version, try to rebuild config in sdb."); + } else { + sz = taosArrayGetSize(taosGetGlobalCfg(tsCfg)); + addArray = taosArrayInit(4, sizeof(SConfigObj)); + for (int i = 0; i < sz; ++i) { + SConfigItem *item = taosArrayGet(taosGetGlobalCfg(tsCfg), i); + obj = sdbAcquire(pMnode->pSdb, SDB_CFG, item->name); + if (obj == NULL) { + SConfigObj *newObj = mndInitConfigObj(item); + if (newObj == NULL) { + code = terrno; + goto _exit; + } + if (NULL == taosArrayPush(addArray, newObj)) { + code = terrno; + goto _exit; + } + } else { + sdbRelease(pMnode->pSdb, obj); + } + } + int32_t addSize = taosArrayGetSize(addArray); + if (addSize > 0) { + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "add-config"); + if (pTrans == NULL) { + code = terrno; + goto _exit; + } + for (int i = 0; i < addSize; ++i) { + SConfigObj *AddObj = taosArrayGet(addArray, i); + if ((code = mndSetCreateConfigCommitLogs(pTrans, AddObj)) != 0) goto _exit; + } + if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _exit; + mInfo("add new config to sdb, nums:%d", addSize); } - mInfo("failed to acquire mnd config version, try to rebuild it , since %s", terrstr()); - goto _OVER; } -_OVER: - return code; +_exit: + if (code != 0) { + mError("failed to try rebuild config in sdb, since %s", tstrerror(code)); + } + sdbRelease(pMnode->pSdb, vObj); + sdbRelease(pMnode->pSdb, obj); + cfgObjArrayCleanUp(addArray); + mndTransDrop(pTrans); + TAOS_RETURN(code); } int32_t mndSetCreateConfigCommitLogs(STrans *pTrans, SConfigObj *item) { @@ -761,6 +802,18 @@ static void cfgArrayCleanUp(SArray *array) { taosArrayDestroy(array); } +static void cfgObjArrayCleanUp(SArray *array) { + if (array == NULL) { + return; + } + int32_t sz = taosArrayGetSize(array); + for (int32_t i = 0; i < sz; ++i) { + SConfigObj *obj = taosArrayGet(array, i); + taosMemoryFree(obj); + } + taosArrayDestroy(array); +} + SArray *initVariablesFromItems(SArray *pItems) { if (pItems == NULL) { return NULL; From cdb4bbc23b4098f388f883a7a39bca032ae8043f Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 12 Dec 2024 13:37:56 +0800 Subject: [PATCH 89/99] Fix macos ut failed. --- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 6372374526..1d89233277 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -60,6 +60,7 @@ static void *dmConfigThreadFp(void *param) { float interval = (curTime - lastTime) / 1000.0f; if (interval >= tsStatusInterval) { dmSendConfigReq(pMgmt); + lastTime = curTime; } } return NULL; From 95a72d14d592367629f66202cb609052122492f5 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 12 Dec 2024 15:07:07 +0800 Subject: [PATCH 90/99] Add en docs. --- .../en/14-reference/01-components/01-taosd.md | 365 +++++++++--------- .../en/14-reference/01-components/02-taosc.md | 162 ++++---- docs/en/14-reference/03-taos-sql/21-node.md | 54 +-- docs/zh/14-reference/03-taos-sql/21-node.md | 4 + 4 files changed, 288 insertions(+), 297 deletions(-) diff --git a/docs/en/14-reference/01-components/01-taosd.md b/docs/en/14-reference/01-components/01-taosd.md index 5dcf5f91ae..77b6dd27e0 100644 --- a/docs/en/14-reference/01-components/01-taosd.md +++ b/docs/en/14-reference/01-components/01-taosd.md @@ -28,67 +28,67 @@ After modifying configuration file parameters, you need to restart the *taosd* s ### Connection Related -|Parameter Name |Supported Version |Description| -|-----------------------|-------------------------|------------| -|firstEp | |Endpoint of the first dnode in the cluster that taosd actively connects to at startup, default value localhost:6030| -|secondEp | |Endpoint of the second dnode in the cluster that taosd tries to connect to if the firstEp is unreachable, no default value| -|fqdn | |The service address that taosd listens on, default is the first hostname configured on the server| -|serverPort | |The port that taosd listens on, default value 6030| -|compressMsgSize | |Whether to compress RPC messages; -1: do not compress any messages; 0: compress all messages; N (N>0): only compress messages larger than N bytes; default value -1| -|shellActivityTimer | |Duration in seconds for the client to send heartbeat to mnode, range 1-120, default value 3 | -|numOfRpcSessions | |Maximum number of connections supported by RPC, range 100-100000, default value 30000| -|numOfRpcThreads | |Number of threads for receiving and sending RPC data, range 1-50, default value is half of the CPU cores| -|numOfTaskQueueThreads | |Number of threads for client to process RPC messages, range 4-16, default value is half of the CPU cores| -|rpcQueueMemoryAllowed | |Maximum memory allowed for received RPC messages in dnode, in bytes, range 104857600-INT64_MAX, default value is 1/10 of server memory | -|resolveFQDNRetryTime | Cancelled after 3.x |Number of retries when FQDN resolution fails| +|Parameter Name |Supported Version |Dynamic Modification|Description| +|-----------------------|-------------------------|--------------------|------------| +|firstEp | |Not supported |Endpoint of the first dnode in the cluster that taosd actively connects to at startup, default value localhost:6030| +|secondEp | |Not supported |Endpoint of the second dnode in the cluster that taosd tries to connect to if the firstEp is unreachable, no default value| +|fqdn | |Not supported |The service address that taosd listens on, default is the first hostname configured on the server| +|serverPort | |Not supported |The port that taosd listens on, default value 6030| +|compressMsgSize | |Supported, effective after restart|Whether to compress RPC messages; -1: do not compress any messages; 0: compress all messages; N (N>0): only compress messages larger than N bytes; default value -1| +|shellActivityTimer | |Supported, effective immediately |Duration in seconds for the client to send heartbeat to mnode, range 1-120, default value 3 | +|numOfRpcSessions | |Supported, effective after restart|Maximum number of connections supported by RPC, range 100-100000, default value 30000| +|numOfRpcThreads | |Supported, effective after restart|Number of threads for receiving and sending RPC data, range 1-50, default value is half of the CPU cores| +|numOfTaskQueueThreads | |Supported, effective after restart|Number of threads for client to process RPC messages, range 4-16, default value is half of the CPU cores| +|rpcQueueMemoryAllowed | |Supported, effective immediately |Maximum memory allowed for received RPC messages in dnode, in bytes, range 104857600-INT64_MAX, default value is 1/10 of server memory | +|resolveFQDNRetryTime | Cancelled after 3.x |Not supported |Number of retries when FQDN resolution fails| |timeToGetAvailableConn | Cancelled after 3.3.4.x |Maximum waiting time to get an available connection, range 10-50000000, in milliseconds, default value 500000| -|maxShellConns | Cancelled after 3.x |Maximum number of connections allowed| -|maxRetryWaitTime | |Maximum timeout for reconnection, default value is 10s| -|shareConnLimit |Added in 3.3.4.0 |Number of requests a connection can share, range 1-512, default value 10| -|readTimeout |Added in 3.3.4.0 |Minimum timeout for a single request, range 64-604800, in seconds, default value 900| +|maxShellConns | Cancelled after 3.x |Supported, effective after restart|Maximum number of connections allowed| +|maxRetryWaitTime | |Supported, effective after restart|Maximum timeout for reconnection, default value is 10s| +|shareConnLimit |Added in 3.3.4.0 |Supported, effective after restart|Number of requests a connection can share, range 1-512, default value 10| +|readTimeout |Added in 3.3.4.0 |Supported, effective after restart|Minimum timeout for a single request, range 64-604800, in seconds, default value 900| ### Monitoring Related -|Parameter Name|Supported Version|Description| -|-----------------------|----------|-| -|monitor | |Whether to collect and report monitoring data, 0: off; 1: on; default value 0| -|monitorFqdn | |The FQDN of the server where the taosKeeper service is located, default value none| -|monitorPort | |The port number listened to by the taosKeeper service, default value 6043| -|monitorInterval | |The time interval for recording system parameters (CPU/memory) in the monitoring database, in seconds, range 1-200000, default value 30| -|monitorMaxLogs | |Number of cached logs pending report| -|monitorComp | |Whether to use compression when reporting monitoring logs| -|monitorLogProtocol | |Whether to print monitoring logs| -|monitorForceV2 | |Whether to use V2 protocol for reporting| -|telemetryReporting | |Whether to upload telemetry, 0: do not upload, 1: upload, default value 1| -|telemetryServer | |Telemetry server address| -|telemetryPort | |Telemetry server port number| -|telemetryInterval | |Telemetry upload interval, in seconds, default 43200| -|crashReporting | |Whether to upload crash information; 0: do not upload, 1: upload; default value 1| +|Parameter Name |Supported Version |Dynamic Modification|Description| +|-----------------------|-------------------------|--------------------|------------| +|monitor | |Supported, effective immediately |Whether to collect and report monitoring data, 0: off; 1: on; default value 0| +|monitorFqdn | |Supported, effective after restart|The FQDN of the server where the taosKeeper service is located, default value none| +|monitorPort | |Supported, effective after restart|The port number listened to by the taosKeeper service, default value 6043| +|monitorInterval | |Supported, effective immediately |The time interval for recording system parameters (CPU/memory) in the monitoring database, in seconds, range 1-200000, default value 30| +|monitorMaxLogs | |Supported, effective immediately |Number of cached logs pending report| +|monitorComp | |Supported, effective after restart|Whether to use compression when reporting monitoring logs| +|monitorLogProtocol | |Supported, effective immediately |Whether to print monitoring logs| +|monitorForceV2 | |Supported, effective immediately |Whether to use V2 protocol for reporting| +|telemetryReporting | |Supported, effective immediately |Whether to upload telemetry, 0: do not upload, 1: upload, default value 1| +|telemetryServer | |Not supported |Telemetry server address| +|telemetryPort | |Not supported |Telemetry server port number| +|telemetryInterval | |Supported, effective immediately |Telemetry upload interval, in seconds, default 43200| +|crashReporting | |Supported, effective immediately |Whether to upload crash information; 0: do not upload, 1: upload; default value 1| ### Query Related -|Parameter Name|Supported Version|Description| -|------------------------|----------|-| -|countAlwaysReturnValue | |Whether count/hyperloglog functions return a value when input data is empty or NULL; 0: return empty row, 1: return; default value 1; When this parameter is set to 1, if the query contains an INTERVAL clause or the query uses TSMA, and the corresponding group or window has empty or NULL data, the corresponding group or window will not return a query result; Note that this parameter should be consistent between client and server| -|tagFilterCache | |Whether to cache tag filter results| -|queryBufferSize | |Not effective yet| -|queryRspPolicy | |Query response strategy| -|filterScalarMode | |Force scalar filter mode, 0: off; 1: on, default value 0| -|queryPlannerTrace | |Internal parameter, whether the query plan outputs detailed logs| -|queryNodeChunkSize | |Internal parameter, chunk size of the query plan| -|queryUseNodeAllocator | |Internal parameter, allocation method of the query plan| -|queryMaxConcurrentTables| |Internal parameter, concurrency number of the query plan| -|queryRsmaTolerance | |Internal parameter, tolerance time for determining which level of rsma data to query, in milliseconds| -|enableQueryHb | |Internal parameter, whether to send query heartbeat messages| -|pqSortMemThreshold | |Internal parameter, memory threshold for sorting| +|Parameter Name |Supported Version |Dynamic Modification|Description| +|-----------------------|-------------------------|--------------------|------------| +|countAlwaysReturnValue | |Supported, effective immediately |Whether count/hyperloglog functions return a value when input data is empty or NULL; 0: return empty row, 1: return; default value 1; When this parameter is set to 1, if the query contains an INTERVAL clause or the query uses TSMA, and the corresponding group or window has empty or NULL data, the corresponding group or window will not return a query result; Note that this parameter should be consistent between client and server| +|tagFilterCache | |Not supported |Whether to cache tag filter results| +|queryBufferSize | |Supported, effective after restart|Not effective yet| +|queryRspPolicy | |Supported, effective immediately |Query response strategy| +|filterScalarMode | |Not supported |Force scalar filter mode, 0: off; 1: on, default value 0| +|queryPlannerTrace | |Supported, effective immediately |Internal parameter, whether the query plan outputs detailed logs| +|queryNodeChunkSize | |Supported, effective immediately |Internal parameter, chunk size of the query plan| +|queryUseNodeAllocator | |Supported, effective immediately |Internal parameter, allocation method of the query plan| +|queryMaxConcurrentTables| |Not supported |Internal parameter, concurrency number of the query plan| +|queryRsmaTolerance | |Not supported |Internal parameter, tolerance time for determining which level of rsma data to query, in milliseconds| +|enableQueryHb | |Supported, effective immediately |Internal parameter, whether to send query heartbeat messages| +|pqSortMemThreshold | |Not supported |Internal parameter, memory threshold for sorting| ### Region Related -|Parameter Name|Supported Version|Description| -|-----------------|----------|-| -|timezone | |Time zone; defaults to dynamically obtaining the current time zone setting from the system| -|locale | |System locale information and encoding format, defaults to obtaining from the system| -|charset | |Character set encoding, defaults to obtaining from the system| +|Parameter Name |Supported Version |Dynamic Modification|Description| +|-----------------------|-------------------------|--------------------|------------| +|timezone | |Not supported |Time zone; defaults to dynamically obtaining the current time zone setting from the system| +|locale | |Not supported |System locale information and encoding format, defaults to obtaining from the system| +|charset | |Not supported |Character set encoding, defaults to obtaining from the system| :::info @@ -166,152 +166,153 @@ The effective value of charset is UTF-8. ### Storage Related -|Parameter Name|Supported Version|Description| -|--------------------|----------|-| -|dataDir | |Directory for data files, all data files are written to this directory, default value /var/lib/taos| -|tempDir | |Specifies the directory for generating temporary files during system operation, default value /tmp| -|minimalDataDirGB | |Minimum space to be reserved in the time-series data storage directory specified by dataDir, in GB, default value 2| -|minimalTmpDirGB | |Minimum space to be reserved in the temporary file directory specified by tempDir, in GB, default value 1| -|minDiskFreeSize |After 3.1.1.0|When the available space on a disk is less than or equal to this threshold, the disk will no longer be selected for generating new data files, unit is bytes, range 52428800-1073741824, default value 52428800; Enterprise parameter| -|s3MigrateIntervalSec|After 3.3.4.3|Trigger cycle for automatic upload of local data files to S3, in seconds. Minimum: 600; Maximum: 100000. Default value 3600; Enterprise parameter| -|s3MigrateEnabled |After 3.3.4.3|Whether to automatically perform S3 migration, default value is 0, which means auto S3 migration is off, can be set to 1; Enterprise parameter| -|s3Accesskey |After 3.3.4.3|Colon-separated user SecretId:SecretKey, for example AKIDsQmwsfKxTo2A6nGVXZN0UlofKn6JRRSJ:lIdoy99ygEacU7iHfogaN2Xq0yumSm1E; Enterprise parameter| -|s3Endpoint |After 3.3.4.3|COS service domain name in the user's region, supports http and https, the region of the bucket must match the endpoint, otherwise it cannot be accessed; Enterprise parameter| -|s3BucketName |After 3.3.4.3|Bucket name, followed by a hyphen and the AppId of the user registered COS service, where AppId is unique to COS, not present in AWS and Alibaba Cloud, needs to be part of the bucket name, separated by a hyphen; parameter values are string type, but do not need quotes; for example test0711-1309024725; Enterprise parameter| -|s3PageCacheSize |After 3.3.4.3|Number of S3 page cache pages, range 4-1048576, unit is pages, default value 4096; Enterprise parameter| -|s3UploadDelaySec |After 3.3.4.3|How long a data file remains unchanged before being uploaded to S3, range 1-2592000 (30 days), in seconds, default value 60; Enterprise parameter| -|cacheLazyLoadThreshold| |Internal parameter, cache loading strategy| +|Parameter Name |Supported Version |Dynamic Modification|Description| +|-----------------------|-------------------------|--------------------|------------| +|dataDir | |Not supported |Directory for data files, all data files are written to this directory, default value /var/lib/taos| +|tempDir | |Not supported |Specifies the directory for generating temporary files during system operation, default value /tmp| +|minimalDataDirGB | |Not supported |Minimum space to be reserved in the time-series data storage directory specified by dataDir, in GB, default value 2| +|minimalTmpDirGB | |Not supported |Minimum space to be reserved in the temporary file directory specified by tempDir, in GB, default value 1| +|minDiskFreeSize |After 3.1.1.0|Supported, effective immediately |When the available space on a disk is less than or equal to this threshold, the disk will no longer be selected for generating new data files, unit is bytes, range 52428800-1073741824, default value 52428800; Enterprise parameter| +|s3MigrateIntervalSec|After 3.3.4.3|Supported, effective immediately |Trigger cycle for automatic upload of local data files to S3, in seconds. Minimum: 600; Maximum: 100000. Default value 3600; Enterprise parameter| +|s3MigrateEnabled |After 3.3.4.3|Supported, effective immediately |Whether to automatically perform S3 migration, default value is 0, which means auto S3 migration is off, can be set to 1; Enterprise parameter| +|s3Accesskey |After 3.3.4.3|Supported, effective after restart|Colon-separated user SecretId:SecretKey, for example AKIDsQmwsfKxTo2A6nGVXZN0UlofKn6JRRSJ:lIdoy99ygEacU7iHfogaN2Xq0yumSm1E; Enterprise parameter| +|s3Endpoint |After 3.3.4.3|Supported, effective after restart|COS service domain name in the user's region, supports http and https, the region of the bucket must match the endpoint, otherwise it cannot be accessed; Enterprise parameter| +|s3BucketName |After 3.3.4.3|Supported, effective after restart|Bucket name, followed by a hyphen and the AppId of the user registered COS service, where AppId is unique to COS, not present in AWS and Alibaba Cloud, needs to be part of the bucket name, separated by a hyphen; parameter values are string type, but do not need quotes; for example test0711-1309024725; Enterprise parameter| +|s3PageCacheSize |After 3.3.4.3|Supported, effective after restart|Number of S3 page cache pages, range 4-1048576, unit is pages, default value 4096; Enterprise parameter| +|s3UploadDelaySec |After 3.3.4.3|Supported, effective immediately |How long a data file remains unchanged before being uploaded to S3, range 1-2592000 (30 days), in seconds, default value 60; Enterprise parameter| +|cacheLazyLoadThreshold| |Supported, effective immediately |Internal parameter, cache loading strategy| ### Cluster Related -|Parameter Name|Supported Version|Description| -|--------------------------|----------|-| -|supportVnodes | |Maximum number of vnodes supported by a dnode, range 0-4096, default value is twice the number of CPU cores + 5| -|numOfCommitThreads | |Maximum number of commit threads, range 0-1024, default value 4| -|numOfMnodeReadThreads | |Number of Read threads for mnode, range 0-1024, default value is one quarter of the CPU cores (not exceeding 4)| -|numOfVnodeQueryThreads | |Number of Query threads for vnode, range 0-1024, default value is twice the number of CPU cores (not exceeding 16)| -|numOfVnodeFetchThreads | |Number of Fetch threads for vnode, range 0-1024, default value is one quarter of the CPU cores (not exceeding 4)| -|numOfVnodeRsmaThreads | |Number of Rsma threads for vnode, range 0-1024, default value is one quarter of the CPU cores (not exceeding 4)| -|numOfQnodeQueryThreads | |Number of Query threads for qnode, range 0-1024, default value is twice the number of CPU cores (not exceeding 16)| -|numOfSnodeSharedThreads | |Number of shared threads for snode, range 0-1024, default value is one quarter of the CPU cores (not less than 2, not exceeding 4)| -|numOfSnodeUniqueThreads | |Number of exclusive threads for snode, range 0-1024, default value is one quarter of the CPU cores (not less than 2, not exceeding 4)| -|ratioOfVnodeStreamThreads | |Ratio of stream computing using vnode threads, range 0.01-4, default value 4| -|ttlUnit | |Unit for ttl parameter, range 1-31572500, in seconds, default value 86400| -|ttlPushInterval | |Frequency of ttl timeout checks, range 1-100000, in seconds, default value 10| -|ttlChangeOnWrite | |Whether ttl expiration time changes with table modification; 0: no change, 1: change; default value 0| -|ttlBatchDropNum | |Number of subtables deleted in a batch for ttl, minimum value 0, default value 10000| -|retentionSpeedLimitMB | |Speed limit for data migration across different levels of disks, range 0-1024, in MB, default value 0, which means no limit| -|maxTsmaNum | |Maximum number of TSMAs that can be created in the cluster; range 0-3; default value 3| -|tmqMaxTopicNum | |Maximum number of topics that can be established for subscription; range 1-10000; default value 20| -|tmqRowSize | |Maximum number of records in a subscription data block, range 1-1000000, default value 4096| -|audit | |Audit feature switch; Enterprise parameter| -|auditInterval | |Time interval for reporting audit data; Enterprise parameter| -|auditCreateTable | |Whether to enable audit feature for creating subtables; Enterprise parameter| -|encryptAlgorithm | |Data encryption algorithm; Enterprise parameter| -|encryptScope | |Encryption scope; Enterprise parameter| -|enableWhiteList | |Switch for whitelist feature; Enterprise parameter| -|syncLogBufferMemoryAllowed| |Maximum memory allowed for sync log cache messages for a dnode, in bytes, range 104857600-INT64_MAX, default value is 1/10 of server memory, effective from versions 3.1.3.2/3.3.2.13| -|syncElectInterval | |Internal parameter, for debugging synchronization module| -|syncHeartbeatInterval | |Internal parameter, for debugging synchronization module| -|syncHeartbeatTimeout | |Internal parameter, for debugging synchronization module| -|syncSnapReplMaxWaitN | |Internal parameter, for debugging synchronization module| -|syncSnapReplMaxWaitN | |Internal parameter, for debugging synchronization module| -|arbHeartBeatIntervalSec | |Internal parameter, for debugging synchronization module| -|arbCheckSyncIntervalSec | |Internal parameter, for debugging synchronization module| -|arbSetAssignedTimeoutSec | |Internal parameter, for debugging synchronization module| -|mndSdbWriteDelta | |Internal parameter, for debugging mnode module| -|mndLogRetention | |Internal parameter, for debugging mnode module| -|skipGrant | |Internal parameter, for authorization checks| -|trimVDbIntervalSec | |Internal parameter, for deleting expired data| -|ttlFlushThreshold | |Internal parameter, frequency of ttl timer| -|compactPullupInterval | |Internal parameter, frequency of data reorganization timer| -|walFsyncDataSizeLimit | |Internal parameter, threshold for WAL to perform FSYNC| -|transPullupInterval | |Internal parameter, retry interval for mnode to execute transactions| -|mqRebalanceInterval | |Internal parameter, interval for consumer rebalancing| -|uptimeInterval | |Internal parameter, for recording system uptime| -|timeseriesThreshold | |Internal parameter, for usage statistics| -|udf | |Whether to start UDF service; 0: do not start, 1: start; default value 0 | -|udfdResFuncs | |Internal parameter, for setting UDF result sets| -|udfdLdLibPath | |Internal parameter, indicates the library path for loading UDF| +|Parameter Name |Supported Version |Dynamic Modification|Description| +|-----------------------|-------------------------|--------------------|------------| +|supportVnodes | |Supported, effective immediately |Maximum number of vnodes supported by a dnode, range 0-4096, default value is twice the number of CPU cores + 5| +|numOfCommitThreads | |Supported, effective after restart|Maximum number of commit threads, range 0-1024, default value 4| +|numOfMnodeReadThreads | |Supported, effective after restart|Number of Read threads for mnode, range 0-1024, default value is one quarter of the CPU cores (not exceeding 4)| +|numOfVnodeQueryThreads | |Supported, effective after restart|Number of Query threads for vnode, range 0-1024, default value is twice the number of CPU cores (not exceeding 16)| +|numOfVnodeFetchThreads | |Supported, effective after restart|Number of Fetch threads for vnode, range 0-1024, default value is one quarter of the CPU cores (not exceeding 4)| +|numOfVnodeRsmaThreads | |Supported, effective after restart|Number of Rsma threads for vnode, range 0-1024, default value is one quarter of the CPU cores (not exceeding 4)| +|numOfQnodeQueryThreads | |Supported, effective after restart|Number of Query threads for qnode, range 0-1024, default value is twice the number of CPU cores (not exceeding 16)| +|numOfSnodeSharedThreads | |Supported, effective after restart|Number of shared threads for snode, range 0-1024, default value is one quarter of the CPU cores (not less than 2, not exceeding 4)| +|numOfSnodeUniqueThreads | |Supported, effective after restart|Number of exclusive threads for snode, range 0-1024, default value is one quarter of the CPU cores (not less than 2, not exceeding 4)| +|ratioOfVnodeStreamThreads | |Supported, effective after restart|Ratio of stream computing using vnode threads, range 0.01-4, default value 4| +|ttlUnit | |Not supported |Unit for ttl parameter, range 1-31572500, in seconds, default value 86400| +|ttlPushInterval | |Supported, effective immediately |Frequency of ttl timeout checks, range 1-100000, in seconds, default value 10| +|ttlChangeOnWrite | |Supported, effective immediately |Whether ttl expiration time changes with table modification; 0: no change, 1: change; default value 0| +|ttlBatchDropNum | |Supported, effective immediately |Number of subtables deleted in a batch for ttl, minimum value 0, default value 10000| +|retentionSpeedLimitMB | |Supported, effective immediately |Speed limit for data migration across different levels of disks, range 0-1024, in MB, default value 0, which means no limit| +|maxTsmaNum | |Supported, effective immediately |Maximum number of TSMAs that can be created in the cluster; range 0-3; default value 3| +|tmqMaxTopicNum | |Supported, effective immediately |Maximum number of topics that can be established for subscription; range 1-10000; default value 20| +|tmqRowSize | |Supported, effective immediately |Maximum number of records in a subscription data block, range 1-1000000, default value 4096| +|audit | |Supported, effective immediately |Audit feature switch; Enterprise parameter| +|auditInterval | |Supported, effective immediately |Time interval for reporting audit data; Enterprise parameter| +|auditCreateTable | |Supported, effective immediately |Whether to enable audit feature for creating subtables; Enterprise parameter| +|encryptAlgorithm | |Not supported |Data encryption algorithm; Enterprise parameter| +|encryptScope | |Not supported |Encryption scope; Enterprise parameter| +|enableWhiteList | |Supported, effective immediately |Switch for whitelist feature; Enterprise parameter| +|syncLogBufferMemoryAllowed| |Supported, effective immediately |Maximum memory allowed for sync log cache messages for a dnode, in bytes, range 104857600-INT64_MAX, default value is 1/10 of server memory, effective from versions 3.1.3.2/3.3.2.13| +|syncElectInterval | |Not supported |Internal parameter, for debugging synchronization module| +|syncHeartbeatInterval | |Not supported |Internal parameter, for debugging synchronization module| +|syncHeartbeatTimeout | |Not supported |Internal parameter, for debugging synchronization module| +|syncSnapReplMaxWaitN | |Supported, effective immediately |Internal parameter, for debugging synchronization module| +|arbHeartBeatIntervalSec | |Supported, effective immediately |Internal parameter, for debugging synchronization module| +|arbCheckSyncIntervalSec | |Supported, effective immediately |Internal parameter, for debugging synchronization module| +|arbSetAssignedTimeoutSec | |Supported, effective immediately |Internal parameter, for debugging synchronization module| +|mndSdbWriteDelta | |Supported, effective immediately |Internal parameter, for debugging mnode module| +|mndLogRetention | |Supported, effective immediately |Internal parameter, for debugging mnode module| +|skipGrant | |Not supported |Internal parameter, for authorization checks| +|trimVDbIntervalSec | |Supported, effective immediately |Internal parameter, for deleting expired data| +|ttlFlushThreshold | |Supported, effective immediately |Internal parameter, frequency of ttl timer| +|compactPullupInterval | |Supported, effective immediately |Internal parameter, frequency of data reorganization timer| +|walFsyncDataSizeLimit | |Supported, effective immediately |Internal parameter, threshold for WAL to perform FSYNC| +|transPullupInterval | |Supported, effective immediately |Internal parameter, retry interval for mnode to execute transactions| +|mqRebalanceInterval | |Supported, effective immediately |Internal parameter, interval for consumer rebalancing| +|uptimeInterval | |Supported, effective immediately |Internal parameter, for recording system uptime| +|timeseriesThreshold | |Supported, effective immediately |Internal parameter, for usage statistics| +|udf | |Supported, effective after restart|Whether to start UDF service; 0: do not start, 1: start; default value 0 | +|udfdResFuncs | |Supported, effective after restart|Internal parameter, for setting UDF result sets| +|udfdLdLibPath | |Supported, effective after restart|Internal parameter, indicates the library path for loading UDF| ### Stream Computing Parameters -| Parameter Name | Supported Version | Description | -|-----------------------|----------|-| -| disableStream | | Switch to enable or disable stream computing | -| streamBufferSize | | Controls the size of the window state cache in memory, default value is 128MB | -| streamAggCnt | | Internal parameter, number of concurrent aggregation computations | -| checkpointInterval | | Internal parameter, checkpoint synchronization interval | -| concurrentCheckpoint | | Internal parameter, whether to check checkpoints concurrently | -| maxStreamBackendCache | | Internal parameter, maximum cache used by stream computing | -| streamSinkDataRate | | Internal parameter, used to control the write speed of stream computing results | +|Parameter Name |Supported Version |Dynamic Modification|Description| +|-----------------------|-------------------------|--------------------|------------| +| disableStream | |Supported, effective immediately | Switch to enable or disable stream computing | +| streamBufferSize | |Supported, effective immediately | Controls the size of the window state cache in memory, default value is 128MB | +| streamAggCnt | |Not supported | Internal parameter, number of concurrent aggregation computations | +| checkpointInterval | |Supported, effective after restart| Internal parameter, checkpoint synchronization interval | +| concurrentCheckpoint | |Supported, effective immediately | Internal parameter, whether to check checkpoints concurrently | +| maxStreamBackendCache | |Supported, effective immediately | Internal parameter, maximum cache used by stream computing | +| streamSinkDataRate | |Supported, effective after restart| Internal parameter, used to control the write speed of stream computing results | ### Log Related -| Parameter Name | Supported Version | Description | -|----------------|----------|-| -| logDir | | Log file directory, operational logs will be written to this directory, default value /var/log/taos | -| minimalLogDirGB | | Stops writing logs when the available space on the disk where the log folder is located is less than this value, unit GB, default value 1 | -| numOfLogLines | | Maximum number of lines allowed in a single log file, default value 10,000,000 | -| asyncLog | | Log writing mode, 0: synchronous, 1: asynchronous, default value 1 | -| logKeepDays | | Maximum retention time for log files, unit: days, default value 0, which means unlimited retention, log files will not be renamed, nor will new log files be rolled out, but the content of the log files may continue to roll depending on the log file size setting; when set to a value greater than 0, when the log file size reaches the set limit, it will be renamed to taosdlog.yyy, where yyy is the timestamp of the last modification of the log file, and a new log file will be rolled out | -| slowLogThreshold| 3.3.3.0 onwards | Slow query threshold, queries taking longer than or equal to this threshold are considered slow, unit seconds, default value 3 | -| slowLogMaxLen | 3.3.3.0 onwards | Maximum length of slow query logs, range 1-16384, default value 4096 | -| slowLogScope | 3.3.3.0 onwards | Type of slow query records, range ALL/QUERY/INSERT/OTHERS/NONE, default value QUERY | -| slowLogExceptDb | 3.3.3.0 onwards | Specifies the database that does not report slow queries, only supports configuring one database | -| debugFlag | | Log switch for running logs, 131 (outputs error and warning logs), 135 (outputs error, warning, and debug logs), 143 (outputs error, warning, debug, and trace logs); default value 131 or 135 (depending on the module) | -| tmrDebugFlag | | Log switch for the timer module, range as above | -| uDebugFlag | | Log switch for the utility module, range as above | -| rpcDebugFlag | | Log switch for the rpc module, range as above | -| qDebugFlag | | Log switch for the query module, range as above | -| dDebugFlag | | Log switch for the dnode module, range as above | -| vDebugFlag | | Log switch for the vnode module, range as above | -| mDebugFlag | | Log switch for the mnode module, range as above | -| azDebugFlag | 3.3.4.3 onwards | Log switch for the S3 module, range as above | -| sDebugFlag | | Log switch for the sync module, range as above | -| tsdbDebugFlag | | Log switch for the tsdb module, range as above | -| tqDebugFlag | | Log switch for the tq module, range as above | -| fsDebugFlag | | Log switch for the fs module, range as above | -| udfDebugFlag | | Log switch for the udf module, range as above | -| smaDebugFlag | | Log switch for the sma module, range as above | -| idxDebugFlag | | Log switch for the index module, range as above | -| tdbDebugFlag | | Log switch for the tdb module, range as above | -| metaDebugFlag | | Log switch for the meta module, range as above | -| stDebugFlag | | Log switch for the stream module, range as above | -| sndDebugFlag | | Log switch for the snode module, range as above | +|Parameter Name |Supported Version |Dynamic Modification|Description| +|-----------------------|-------------------------|--------------------|------------| +| logDir | |Not supported | Log file directory, operational logs will be written to this directory, default value /var/log/taos | +| minimalLogDirGB | |Not supported | Stops writing logs when the available space on the disk where the log folder is located is less than this value, unit GB, default value 1 | +| numOfLogLines | |Supported, effective immediately | Maximum number of lines allowed in a single log file, default value 10,000,000 | +| asyncLog | |Supported, effective immediately | Log writing mode, 0: synchronous, 1: asynchronous, default value 1 | +| logKeepDays | |Supported, effective immediately | Maximum retention time for log files, unit: days, default value 0, which means unlimited retention, log files will not be renamed, nor will new log files be rolled out, but the content of the log files may continue to roll depending on the log file size setting; when set to a value greater than 0, when the log file size reaches the set limit, it will be renamed to taosdlog.yyy, where yyy is the timestamp of the last modification of the log file, and a new log file will be rolled out | +| slowLogThreshold| 3.3.3.0 onwards |Supported, effective immediately | Slow query threshold, queries taking longer than or equal to this threshold are considered slow, unit seconds, default value 3 | +| slowLogMaxLen | 3.3.3.0 onwards |Supported, effective immediately | Maximum length of slow query logs, range 1-16384, default value 4096 | +| slowLogScope | 3.3.3.0 onwards |Supported, effective immediately | Type of slow query records, range ALL/QUERY/INSERT/OTHERS/NONE, default value QUERY | +| slowLogExceptDb | 3.3.3.0 onwards |Supported, effective immediately | Specifies the database that does not report slow queries, only supports configuring one database | +| debugFlag | |Supported, effective immediately | Log switch for running logs, 131 (outputs error and warning logs), 135 (outputs error, warning, and debug logs), 143 (outputs error, warning, debug, and trace logs); default value 131 or 135 (depending on the module) | +| tmrDebugFlag | |Supported, effective immediately | Log switch for the timer module, range as above | +| uDebugFlag | |Supported, effective immediately | Log switch for the utility module, range as above | +| rpcDebugFlag | |Supported, effective immediately | Log switch for the rpc module, range as above | +| qDebugFlag | |Supported, effective immediately | Log switch for the query module, range as above | +| dDebugFlag | |Supported, effective immediately | Log switch for the dnode module, range as above | +| vDebugFlag | |Supported, effective immediately | Log switch for the vnode module, range as above | +| mDebugFlag | |Supported, effective immediately | Log switch for the mnode module, range as above | +| azDebugFlag | 3.3.4.3 onwards |Supported, effective immediately | Log switch for the S3 module, range as above | +| sDebugFlag | |Supported, effective immediately | Log switch for the sync module, range as above | +| tsdbDebugFlag | |Supported, effective immediately | Log switch for the tsdb module, range as above | +| tqDebugFlag | |Supported, effective immediately | Log switch for the tq module, range as above | +| fsDebugFlag | |Supported, effective immediately | Log switch for the fs module, range as above | +| udfDebugFlag | |Supported, effective immediately | Log switch for the udf module, range as above | +| smaDebugFlag | |Supported, effective immediately | Log switch for the sma module, range as above | +| idxDebugFlag | |Supported, effective immediately | Log switch for the index module, range as above | +| tdbDebugFlag | |Supported, effective immediately | Log switch for the tdb module, range as above | +| metaDebugFlag | |Supported, effective immediately | Log switch for the meta module, range as above | +| stDebugFlag | |Supported, effective immediately | Log switch for the stream module, range as above | +| sndDebugFlag | |Supported, effective immediately | Log switch for the snode module, range as above | ### Debugging Related -| Parameter Name | Supported Version | Description | -|----------------------|-------------------|-------------| -| enableCoreFile | | Whether to generate a core file when crashing, 0: do not generate, 1: generate; default value is 1 | -| configDir | | Directory where the configuration files are located | -| scriptDir | | Directory for internal test tool scripts | -| assert | | Assertion control switch, default value is 0 | -| randErrorChance | | Internal parameter, used for random failure testing | -| randErrorDivisor | | Internal parameter, used for random failure testing | -| randErrorScope | | Internal parameter, used for random failure testing | -| safetyCheckLevel | | Internal parameter, used for random failure testing | -| experimental | | Internal parameter, used for some experimental features | -| simdEnable | After 3.3.4.3 | Internal parameter, used for testing SIMD acceleration | -| AVX512Enable | After 3.3.4.3 | Internal parameter, used for testing AVX512 acceleration | -| rsyncPort | | Internal parameter, used for debugging stream computing | -| snodeAddress | | Internal parameter, used for debugging stream computing | -| checkpointBackupDir | | Internal parameter, used for restoring snode data | -| enableAuditDelete | | Internal parameter, used for testing audit functions | -| slowLogThresholdTest | | Internal parameter, used for testing slow logs | +|Parameter Name |Supported Version |Dynamic Modification|Description| +|-----------------------|-------------------------|--------------------|------------| +| enableCoreFile | |Supported, effective immediately | Whether to generate a core file when crashing, 0: do not generate, 1: generate; default value is 1 | +| configDir | |Not supported | Directory where the configuration files are located | +|forceReadConfig | |Not supported ||Force the use of parameters from the configuration file,default value: 0| +| scriptDir | |Not supported | Directory for internal test tool scripts | +| assert | |Not supported | Assertion control switch, default value is 0 | +| randErrorChance | |Supported, effective immediately | Internal parameter, used for random failure testing | +| randErrorDivisor | |Supported, effective immediately | Internal parameter, used for random failure testing | +| randErrorScope | |Supported, effective immediately | Internal parameter, used for random failure testing | +| safetyCheckLevel | |Supported, effective immediately | Internal parameter, used for random failure testing | +| experimental | |Supported, effective immediately | Internal parameter, used for some experimental features | +| simdEnable | After 3.3.4.3 |Not supported | Internal parameter, used for testing SIMD acceleration | +| AVX512Enable | After 3.3.4.3 |Not supported | Internal parameter, used for testing AVX512 acceleration | +| rsyncPort | |Not supported | Internal parameter, used for debugging stream computing | +| snodeAddress | |Supported, effective immediately | Internal parameter, used for debugging stream computing | +| checkpointBackupDir | |Supported, effective immediately | Internal parameter, used for restoring snode data | +| enableAuditDelete | |Not supported | Internal parameter, used for testing audit functions | +| slowLogThresholdTest | |Not supported | Internal parameter, used for testing slow logs | +| bypassFlag |After 3.3.4.5 |Supported, effective immediately | Internal parameter, used for short-circuit testing| ### Compression Parameters -| Parameter Name | Supported Version | Description | -|----------------|-------------------|-------------| -| fPrecision | | Sets the compression precision for float type floating numbers, range 0.1 ~ 0.00000001, default value 0.00000001, floating numbers smaller than this value will have their mantissa truncated | -| dPrecision | | Sets the compression precision for double type floating numbers, range 0.1 ~ 0.0000000000000001, default value 0.0000000000000001, floating numbers smaller than this value will have their mantissa truncated | -| lossyColumn | Before 3.3.0.0 | Enables TSZ lossy compression for float and/or double types; range float/double/none; default value none, indicating lossless compression is off | -| ifAdtFse | | When TSZ lossy compression is enabled, use the FSE algorithm instead of the HUFFMAN algorithm, FSE algorithm is faster in compression but slightly slower in decompression, choose this for faster compression speed; 0: off, 1: on; default value is 0 | -| maxRange | | Internal parameter, used for setting lossy compression | -| curRange | | Internal parameter, used for setting lossy compression | -| compressor | | Internal parameter, used for setting lossy compression | +|Parameter Name |Supported Version |Dynamic Modification|Description| +|-----------------------|-------------------------|--------------------|------------| +| fPrecision | |Supported, effective immediately | Sets the compression precision for float type floating numbers, range 0.1 ~ 0.00000001, default value 0.00000001, floating numbers smaller than this value will have their mantissa truncated | +| dPrecision | |Supported, effective immediately | Sets the compression precision for double type floating numbers, range 0.1 ~ 0.0000000000000001, default value 0.0000000000000001, floating numbers smaller than this value will have their mantissa truncated | +| lossyColumn | Before 3.3.0.0 |Not supported | Enables TSZ lossy compression for float and/or double types; range float/double/none; default value none, indicating lossless compression is off | +| ifAdtFse | |Supported, effective after restart| When TSZ lossy compression is enabled, use the FSE algorithm instead of the HUFFMAN algorithm, FSE algorithm is faster in compression but slightly slower in decompression, choose this for faster compression speed; 0: off, 1: on; default value is 0 | +| maxRange | |Supported, effective after restart| Internal parameter, used for setting lossy compression | +| curRange | |Supported, effective after restart| Internal parameter, used for setting lossy compression | +| compressor | |Supported, effective after restart| Internal parameter, used for setting lossy compression | **Additional Notes** diff --git a/docs/en/14-reference/01-components/02-taosc.md b/docs/en/14-reference/01-components/02-taosc.md index 46ee43ce81..7b8d3ae78c 100644 --- a/docs/en/14-reference/01-components/02-taosc.md +++ b/docs/en/14-reference/01-components/02-taosc.md @@ -10,107 +10,109 @@ The TDengine client driver provides all the APIs needed for application programm ### Connection Related -|Parameter Name|Supported Version|Description| -|----------------------|----------|-------------| -|firstEp | |At startup, the endpoint of the first dnode in the cluster to actively connect to, default value: hostname:6030, if the server's hostname cannot be obtained, it is assigned to localhost| -|secondEp | |At startup, if the firstEp cannot be connected, try to connect to the endpoint of the second dnode in the cluster, no default value| -|compressMsgSize | |Whether to compress RPC messages; -1: no messages are compressed; 0: all messages are compressed; N (N>0): only messages larger than N bytes are compressed; default value -1| -|shellActivityTimer | |The duration in seconds for the client to send heartbeats to mnode, range 1-120, default value 3| -|numOfRpcSessions | |Maximum number of connections supported by RPC, range 100-100000, default value 30000| -|numOfRpcThreads | |Number of threads for RPC to send and receive data, range 1-50, default value is half of the CPU cores| -|numOfTaskQueueThreads | |Number of threads for the client to handle RPC messages, range 4-16, default value is half of the CPU cores| -|timeToGetAvailableConn| Cancelled after 3.3.4.* |The longest waiting time to get an available connection, range 10-50000000, in milliseconds, default value 500000| -|useAdapter | |Internal parameter, whether to use taosadapter, affects CSV file import| -|shareConnLimit |Added in 3.3.4.0|Internal parameter, the number of queries a link can share, range 1-256, default value 10| -|readTimeout |Added in 3.3.4.0|Internal parameter, minimum timeout, range 64-604800, in seconds, default value 900| +|Parameter Name|Supported Version|Dynamic Modification|Description| +|----------------------|----------|--------------------|-------------| +|firstEp | |Supported, effective immediately |At startup, the endpoint of the first dnode in the cluster to actively connect to, default value: hostname:6030, if the server's hostname cannot be obtained, it is assigned to localhost| +|secondEp | |Supported, effective immediately |At startup, if the firstEp cannot be connected, try to connect to the endpoint of the second dnode in the cluster, no default value| +|compressMsgSize | |Supported, effective immediately |Whether to compress RPC messages; -1: no messages are compressed; 0: all messages are compressed; N (N>0): only messages larger than N bytes are compressed; default value -1| +|shellActivityTimer | |Not supported |The duration in seconds for the client to send heartbeats to mnode, range 1-120, default value 3| +|numOfRpcSessions | |Supported, effective immediately |Maximum number of connections supported by RPC, range 100-100000, default value 30000| +|numOfRpcThreads | |Not supported |Number of threads for RPC to send and receive data, range 1-50, default value is half of the CPU cores| +|numOfTaskQueueThreads | |Not supported |Number of threads for the client to handle RPC messages, range 4-16, default value is half of the CPU cores| +|timeToGetAvailableConn| Cancelled after 3.3.4.* |Not supported |The longest waiting time to get an available connection, range 10-50000000, in milliseconds, default value 500000| +|useAdapter | |Supported, effective immediately |Internal parameter, whether to use taosadapter, affects CSV file import| +|shareConnLimit |Added in 3.3.4.0|Not supported |Internal parameter, the number of queries a link can share, range 1-256, default value 10| +|readTimeout |Added in 3.3.4.0|Not supported |Internal parameter, minimum timeout, range 64-604800, in seconds, default value 900| ### Query Related -|Parameter Name|Supported Version|Description| -|---------------------------------|---------|-| -|countAlwaysReturnValue | |Whether the count/hyperloglog function returns a value when the input data is empty or NULL; 0: returns an empty row, 1: returns; default value 1; when this parameter is set to 1, if the query contains an INTERVAL clause or the query uses TSMA, and the corresponding group or window has empty or NULL data, the corresponding group or window will not return a query result; note that this parameter should be consistent between client and server| -|keepColumnName | |Automatically sets the alias to the column name (excluding the function name) when querying with Last, First, LastRow functions without specifying an alias, thus the order by clause will automatically refer to the column corresponding to the function; 1: automatically sets the alias to the column name (excluding the function name), 0: does not automatically set an alias; default value: 0| -|multiResultFunctionStarReturnTags|After 3.3.3.0|When querying a supertable, whether last(\*)/last_row(\*)/first(\*) returns tag columns; when querying basic tables, subtables, it is not affected by this parameter; 0: does not return tag columns, 1: returns tag columns; default value: 0; when this parameter is set to 0, last(\*)/last_row(\*)/first(\*) only returns the ordinary columns of the supertable; when set to 1, it returns both the ordinary columns and tag columns of the supertable| -|metaCacheMaxSize | |Specifies the maximum size of metadata cache for a single client, in MB; default value -1, meaning unlimited| -|maxTsmaCalcDelay | |The allowable delay for tsma calculation by the client during query, range 600s - 86400s, i.e., 10 minutes - 1 day; default value: 600 seconds| -|tsmaDataDeleteMark | |The retention time for intermediate results of historical data calculated by TSMA, in milliseconds; range >= 3600000, i.e., at least 1h; default value: 86400000, i.e., 1d | -|queryPolicy | |Execution strategy for query statements, 1: only use vnode, do not use qnode; 2: subtasks without scan operators are executed on qnode, subtasks with scan operators are executed on vnode; 3: vnode only runs scan operators, all other operators are executed on qnode; default value: 1| -|queryTableNotExistAsEmpty | |Whether to return an empty result set when the queried table does not exist; false: returns an error; true: returns an empty result set; default value false| -|querySmaOptimize | |Optimization strategy for sma index, 0: do not use sma index, always query from original data; 1: use sma index, directly query from pre-calculated results for eligible statements; default value: 0| -|queryPlannerTrace | |Internal parameter, whether the query plan outputs detailed logs| -|queryNodeChunkSize | |Internal parameter, chunk size of the query plan| -|queryUseNodeAllocator | |Internal parameter, allocation method of the query plan| -|queryMaxConcurrentTables | |Internal parameter, concurrency number of the query plan| -|enableQueryHb | |Internal parameter, whether to send query heartbeat messages| -|minSlidingTime | |Internal parameter, minimum allowable value for sliding| -|minIntervalTime | |Internal parameter, minimum allowable value for interval| +|Parameter Name|Supported Version|Dynamic Modification|Description| +|----------------------|----------|--------------------|-------------| +|countAlwaysReturnValue | |Supported, effective immediately |Whether the count/hyperloglog function returns a value when the input data is empty or NULL; 0: returns an empty row, 1: returns; default value 1; when this parameter is set to 1, if the query contains an INTERVAL clause or the query uses TSMA, and the corresponding group or window has empty or NULL data, the corresponding group or window will not return a query result; note that this parameter should be consistent between client and server| +|keepColumnName | |Supported, effective immediately |Automatically sets the alias to the column name (excluding the function name) when querying with Last, First, LastRow functions without specifying an alias, thus the order by clause will automatically refer to the column corresponding to the function; 1: automatically sets the alias to the column name (excluding the function name), 0: does not automatically set an alias; default value: 0| +|multiResultFunctionStarReturnTags|After 3.3.3.0|Supported, effective immediately |When querying a supertable, whether last(\*)/last_row(\*)/first(\*) returns tag columns; when querying basic tables, subtables, it is not affected by this parameter; 0: does not return tag columns, 1: returns tag columns; default value: 0; when this parameter is set to 0, last(\*)/last_row(\*)/first(\*) only returns the ordinary columns of the supertable; when set to 1, it returns both the ordinary columns and tag columns of the supertable| +|metaCacheMaxSize | |Supported, effective immediately |Specifies the maximum size of metadata cache for a single client, in MB; default value -1, meaning unlimited| +|maxTsmaCalcDelay | |Supported, effective immediately |The allowable delay for tsma calculation by the client during query, range 600s - 86400s, i.e., 10 minutes - 1 day; default value: 600 seconds| +|tsmaDataDeleteMark | |Supported, effective immediately |The retention time for intermediate results of historical data calculated by TSMA, in milliseconds; range >= 3600000, i.e., at least 1h; default value: 86400000, i.e., 1d | +|queryPolicy | |Supported, effective immediately |Execution strategy for query statements, 1: only use vnode, do not use qnode; 2: subtasks without scan operators are executed on qnode, subtasks with scan operators are executed on vnode; 3: vnode only runs scan operators, all other operators are executed on qnode; default value: 1| +|queryTableNotExistAsEmpty | |Supported, effective immediately |Whether to return an empty result set when the queried table does not exist; false: returns an error; true: returns an empty result set; default value false| +|querySmaOptimize | |Supported, effective immediately |Optimization strategy for sma index, 0: do not use sma index, always query from original data; 1: use sma index, directly query from pre-calculated results for eligible statements; default value: 0| +|queryPlannerTrace | |Supported, effective immediately |Internal parameter, whether the query plan outputs detailed logs| +|queryNodeChunkSize | |Supported, effective immediately |Internal parameter, chunk size of the query plan| +|queryUseNodeAllocator | |Supported, effective immediately |Internal parameter, allocation method of the query plan| +|queryMaxConcurrentTables | |Not supported |Internal parameter, concurrency number of the query plan| +|enableQueryHb | |Supported, effective immediately |Internal parameter, whether to send query heartbeat messages| +|minSlidingTime | |Supported, effective immediately |Internal parameter, minimum allowable value for sliding| +|minIntervalTime | |Supported, effective immediately |Internal parameter, minimum allowable value for interval| ### Writing Related -| Parameter Name | Supported Version | Description | -|---------------------------------|-------------------|-------------| -| smlChildTableName | | Key for custom child table name in schemaless, no default value | -| smlAutoChildTableNameDelimiter | | Delimiter between schemaless tags, concatenated as the child table name, no default value | -| smlTagName | | Default tag name when schemaless tag is empty, default value "_tag_null" | -| smlTsDefaultName | | Configuration for setting the time column name in schemaless auto table creation, default value "_ts" | -| smlDot2Underline | | Converts dots in supertable names to underscores in schemaless | -| maxInsertBatchRows | | Internal parameter, maximum number of rows per batch insert | +|Parameter Name|Supported Version|Dynamic Modification|Description| +|----------------------|----------|--------------------|-------------| +| smlChildTableName | |Supported, effective immediately | Key for custom child table name in schemaless, no default value | +| smlAutoChildTableNameDelimiter | |Supported, effective immediately | Delimiter between schemaless tags, concatenated as the child table name, no default value | +| smlTagName | |Supported, effective immediately | Default tag name when schemaless tag is empty, default value "_tag_null" | +| smlTsDefaultName | |Supported, effective immediately | Configuration for setting the time column name in schemaless auto table creation, default value "_ts" | +| smlDot2Underline | |Supported, effective immediately | Converts dots in supertable names to underscores in schemaless | +| maxInsertBatchRows | |Supported, effective immediately | Internal parameter, maximum number of rows per batch insert | ### Region Related -| Parameter Name | Supported Version | Description | -|----------------|-------------------|-------------| -| timezone | | Time zone; defaults to dynamically obtaining the current system time zone setting | -| locale | | System locale and encoding format, defaults to system settings | -| charset | | Character set encoding, defaults to system settings | +|Parameter Name|Supported Version|Dynamic Modification|Description| +|----------------------|----------|--------------------|-------------| +| timezone | |Supported, effective immediately | Time zone; defaults to dynamically obtaining the current system time zone setting | +| locale | |Supported, effective immediately | System locale and encoding format, defaults to system settings | +| charset | |Supported, effective immediately | Character set encoding, defaults to system settings | ### Storage Related -| Parameter Name | Supported Version | Description | -|-----------------|-------------------|-------------| -| tempDir | | Specifies the directory for generating temporary files during operation, default on Linux platform is /tmp | -| minimalTmpDirGB | | Minimum space required to be reserved in the directory specified by tempDir, in GB, default value: 1 | +|Parameter Name|Supported Version|Dynamic Modification|Description| +|----------------------|----------|--------------------|-------------| +| tempDir | |Supported, effective immediately | Specifies the directory for generating temporary files during operation, default on Linux platform is /tmp | +| minimalTmpDirGB | |Supported, effective immediately | Minimum space required to be reserved in the directory specified by tempDir, in GB, default value: 1 | ### Log Related -| Parameter Name | Supported Version | Description | -|------------------|-------------------|-------------| -| logDir | | Log file directory, operational logs will be written to this directory, default value: /var/log/taos | -| minimalLogDirGB | | Stops writing logs when the disk space available in the log directory is less than this value, in GB, default value: 1 | -| numOfLogLines | | Maximum number of lines allowed in a single log file, default value: 10,000,000 | -| asyncLog | | Log writing mode, 0: synchronous, 1: asynchronous, default value: 1 | -| logKeepDays | | Maximum retention time for log files, in days, default value: 0, meaning unlimited retention. Log files will not be renamed, nor will new log files be rolled out, but the content of the log files may continue to roll depending on the log file size setting; when set to a value greater than 0, the log file will be renamed to taoslogx.yyy, where yyy is the timestamp of the last modification of the log file, and a new log file will be rolled out | -| debugFlag | | Log switch for running logs, 131 (output error and warning logs), 135 (output error, warning, and debug logs), 143 (output error, warning, debug, and trace logs); default value 131 or 135 (depending on the module) | -| tmrDebugFlag | | Log switch for the timer module, value range as above | -| uDebugFlag | | Log switch for the utility module, value range as above | -| rpcDebugFlag | | Log switch for the rpc module, value range as above | -| jniDebugFlag | | Log switch for the jni module, value range as above | -| qDebugFlag | | Log switch for the query module, value range as above | -| cDebugFlag | | Log switch for the client module, value range as above | -| simDebugFlag | | Internal parameter, log switch for the test tool, value range as above | -| tqClientDebugFlag| After 3.3.4.3 | Log switch for the client module, value range as above | +|Parameter Name|Supported Version|Dynamic Modification|Description| +|----------------------|----------|--------------------|-------------| +| logDir | |Not supported | Log file directory, operational logs will be written to this directory, default value: /var/log/taos | +| minimalLogDirGB | |Supported, effective immediately | Stops writing logs when the disk space available in the log directory is less than this value, in GB, default value: 1 | +| numOfLogLines | |Supported, effective immediately | Maximum number of lines allowed in a single log file, default value: 10,000,000 | +| asyncLog | |Supported, effective immediately | Log writing mode, 0: synchronous, 1: asynchronous, default value: 1 | +| logKeepDays | |Supported, effective immediately | Maximum retention time for log files, in days, default value: 0, meaning unlimited retention. Log files will not be renamed, nor will new log files be rolled out, but the content of the log files may continue to roll depending on the log file size setting; when set to a value greater than 0, the log file will be renamed to taoslogx.yyy, where yyy is the timestamp of the last modification of the log file, and a new log file will be rolled out | +| debugFlag | |Supported, effective immediately | Log switch for running logs, 131 (output error and warning logs), 135 (output error, warning, and debug logs), 143 (output error, warning, debug, and trace logs); default value 131 or 135 (depending on the module) | +| tmrDebugFlag | |Supported, effective immediately | Log switch for the timer module, value range as above | +| uDebugFlag | |Supported, effective immediately | Log switch for the utility module, value range as above | +| rpcDebugFlag | |Supported, effective immediately | Log switch for the rpc module, value range as above | +| jniDebugFlag | |Supported, effective immediately | Log switch for the jni module, value range as above | +| qDebugFlag | |Supported, effective immediately | Log switch for the query module, value range as above | +| cDebugFlag | |Supported, effective immediately | Log switch for the client module, value range as above | +| simDebugFlag | |Supported, effective immediately | Internal parameter, log switch for the test tool, value range as above | +| tqClientDebugFlag| After 3.3.4.3 |Supported, effective immediately | Log switch for the client module, value range as above | ### Debugging Related -| Parameter Name | Supported Version | Description | -|------------------|-------------------|-------------| -| crashReporting | | Whether to upload crash to telemetry, 0: do not upload, 1: upload; default value: 1 | -| enableCoreFile | | Whether to generate a core file when crashing, 0: do not generate, 1: generate; default value: 1 | -| assert | | Assertion control switch, default value: 0 | -| configDir | | Directory for configuration files | -| scriptDir | | Internal parameter, directory for test cases | -| randErrorChance | After 3.3.3.0 | Internal parameter, used for random failure testing | -| randErrorDivisor | After 3.3.3.0 | Internal parameter, used for random failure testing | -| randErrorScope | After 3.3.3.0 | Internal parameter, used for random failure testing | -| safetyCheckLevel | After 3.3.3.0 | Internal parameter, used for random failure testing | -| simdEnable | After 3.3.4.3 | Internal parameter, used for testing SIMD acceleration | -| AVX512Enable | After 3.3.4.3 | Internal parameter, used for testing AVX512 acceleration | +|Parameter Name|Supported Version|Dynamic Modification|Description| +|----------------------|----------|--------------------|-------------| +| crashReporting | |Supported, effective immediately | Whether to upload crash to telemetry, 0: do not upload, 1: upload; default value: 1 | +| enableCoreFile | |Supported, effective immediately | Whether to generate a core file when crashing, 0: do not generate, 1: generate; default value: 1 | +| assert | |Not supported | Assertion control switch, default value: 0 | +| configDir | |Not supported | Directory for configuration files | +| scriptDir | |Not supported | Internal parameter, directory for test cases | +| randErrorChance | After 3.3.3.0 |Not supported | Internal parameter, used for random failure testing | +| randErrorDivisor | After 3.3.3.0 |Not supported | Internal parameter, used for random failure testing | +| randErrorScope | After 3.3.3.0 |Not supported | Internal parameter, used for random failure testing | +| safetyCheckLevel | After 3.3.3.0 |Not supported | Internal parameter, used for random failure testing | +| simdEnable | After 3.3.4.3 |Not supported | Internal parameter, used for testing SIMD acceleration | +| AVX512Enable | After 3.3.4.3 |Not supported | Internal parameter, used for testing AVX512 acceleration | +| bypassFlag |After 3.3.4.5 |Supported, effective immediately | Internal parameter, used for short-circuit testing| + ### SHELL Related -|Parameter Name|Supported Version|Description| -|-----------------|----------|-| -|enableScience | |Whether to enable scientific notation for displaying floating numbers; 0: do not enable, 1: enable; default value: 1| +|Parameter Name|Supported Version|Dynamic Modification|Description| +|----------------------|----------|--------------------|-------------| +|enableScience | |Not supported |Whether to enable scientific notation for displaying floating numbers; 0: do not enable, 1: enable; default value: 1| ## API diff --git a/docs/en/14-reference/03-taos-sql/21-node.md b/docs/en/14-reference/03-taos-sql/21-node.md index 1da0117a89..76cb68e9be 100644 --- a/docs/en/14-reference/03-taos-sql/21-node.md +++ b/docs/en/14-reference/03-taos-sql/21-node.md @@ -41,38 +41,28 @@ If there is a single replica on the node and the node is offline, to forcibly de ALTER DNODE dnode_id dnode_option ALTER ALL DNODES dnode_option - -dnode_option: { - 'resetLog' - | 'balance' 'value' - | 'monitor' 'value' - | 'debugFlag' 'value' - | 'monDebugFlag' 'value' - | 'vDebugFlag' 'value' - | 'mDebugFlag' 'value' - | 'cDebugFlag' 'value' - | 'httpDebugFlag' 'value' - | 'qDebugflag' 'value' - | 'sdbDebugFlag' 'value' - | 'uDebugFlag' 'value' - | 'tsdbDebugFlag' 'value' - | 'sDebugflag' 'value' - | 'rpcDebugFlag' 'value' - | 'dDebugFlag' 'value' - | 'mqttDebugFlag' 'value' - | 'wDebugFlag' 'value' - | 'tmrDebugFlag' 'value' - | 'cqDebugFlag' 'value' -} ``` -The modifiable configuration items in the syntax above are configured in the same way as in the dnode configuration file, the difference being that modifications are dynamic, take immediate effect, and do not require restarting the dnode. +For configuration parameters that support dynamic modification, you can use the ALTER DNODE or ALTER ALL DNODES syntax to modify the values of configuration parameters in a dnode. Starting from version 3.3.4.0, the modified configuration parameters will be automatically persisted and will remain effective even after the database service is restarted. -`value` is the value of the parameter, which needs to be in string format. For example, to change the log output level of dnode 1 to debug: +To check whether a configuration parameter supports dynamic modification, please refer to the following page: [taosd Reference](../01-components/01-taosd.md) + +The value is the parameter's value and needs to be in character format. For example, to change the log output level of dnode 1 to debug: ```sql ALTER DNODE 1 'debugFlag' '143'; ``` +### Additional Notes: +Configuration parameters in a dnode are divided into global configuration parameters and local configuration parameters. You can check the category field in SHOW VARIABLES or SHOW DNODE dnode_id VARIABLE to determine whether a configuration parameter is a global configuration parameter or a local configuration parameter: + +Local configuration parameters: You can use ALTER DNODE or ALTER ALL DNODES to update the local configuration parameters of a specific dnode or all dnodes. +Global configuration parameters: Global configuration parameters require consistency across all dnodes, so you can only use ALTER ALL DNODES to update the global configuration parameters of all dnodes. +There are three cases for whether a configuration parameter can be dynamically modified: + +Supports dynamic modification, effective immediately +Supports dynamic modification, effective after restart +Does not support dynamic modification +For configuration parameters that take effect after a restart, you can see the modified values through SHOW VARIABLES or SHOW DNODE dnode_id VARIABLE, but you need to restart the database service to make them effective. ## Add Management Node @@ -136,18 +126,12 @@ If the client is also considered as part of the cluster in a broader sense, the ```sql ALTER LOCAL local_option - -local_option: { - 'resetLog' - | 'rpcDebugFlag' 'value' - | 'tmrDebugFlag' 'value' - | 'cDebugFlag' 'value' - | 'uDebugFlag' 'value' - | 'debugFlag' 'value' -} ``` -The parameters in the syntax above are used in the same way as in the configuration file for the client, but do not require a restart of the client, and the changes take effect immediately. +You can use the above syntax to modify the client's configuration parameters, and there is no need to restart the client. The changes take effect immediately. + +To check whether a configuration parameter supports dynamic modification, please refer to the following page:[taosc Reference](../01-components/02-taosc.md) + ## View Client Configuration diff --git a/docs/zh/14-reference/03-taos-sql/21-node.md b/docs/zh/14-reference/03-taos-sql/21-node.md index 43d805d90e..f86dc6a86c 100644 --- a/docs/zh/14-reference/03-taos-sql/21-node.md +++ b/docs/zh/14-reference/03-taos-sql/21-node.md @@ -126,6 +126,10 @@ SHOW CLUSTER ALIVE; 如果将客户端也看作广义的集群的一部分，可以通过如下命令动态修改客户端配置参数。 +```sql +ALTER LOCAL local_option +``` + 您可以使用以上语法更该客户端的配置参数，并且不需要重启客户端，修改后立即生效。 对于一个配置参数是否支持动态修改，请您参考以下页面：[taosc 参考手册](../01-components/02-taosc.md) From d6078fd0b7e41ecb1603113017cf81ed375f43a4 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 12 Dec 2024 17:40:35 +0800 Subject: [PATCH 91/99] Enh use tmsg to encode/decode global config. --- source/dnode/mnode/impl/inc/mndDef.h | 9 +- source/dnode/mnode/impl/src/mndConfig.c | 253 ++++++++++-------------- source/dnode/mnode/impl/src/mndDef.c | 97 ++++++++- 3 files changed, 199 insertions(+), 160 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index d2ab5d9687..0656bf8dc2 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -324,13 +324,16 @@ typedef struct { float fval; int32_t i32; int64_t i64; - char str[TSDB_CONFIG_VALUE_LEN]; + char* str; }; } SConfigObj; +int32_t tEncodeSConfigObj(SEncoder* pEncoder, const SConfigObj* pObj); +int32_t tDecodeSConfigObj(SDecoder* pDecoder, SConfigObj* pObj); SConfigObj* mndInitConfigObj(SConfigItem* pItem); SConfigObj* mndInitConfigVersion(); int32_t mndUpdateObj(SConfigObj* pObj, const char* name, char* value); +void tFreeSConfigObj(SConfigObj* obj); typedef struct { int32_t maxUsers; @@ -505,8 +508,8 @@ typedef struct { int64_t dstTbUid; int8_t intervalUnit; int8_t slidingUnit; - int8_t timezone; // int8_t is not enough, timezone is unit of second - int32_t dstVgId; // for stream + int8_t timezone; // int8_t is not enough, timezone is unit of second + int32_t dstVgId; // for stream int64_t interval; int64_t offset; int64_t sliding; diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index a18e524581..84a9110bc5 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -27,7 +27,6 @@ #define CFG_ALTER_TIMEOUT 3 * 1000 static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pInMCfgReq, int32_t optLen, int32_t *pOutValue); -static int32_t cfgUpdateItem(SConfigItem *pItem, SConfigObj *obj); static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq); static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq); static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp); @@ -66,116 +65,105 @@ int32_t mndInitConfig(SMnode *pMnode) { SSdbRaw *mnCfgActionEncode(SConfigObj *obj) { int32_t code = 0; int32_t lino = 0; - terrno = TSDB_CODE_OUT_OF_MEMORY; - char buf[30]; + void *buf = NULL; - int32_t sz = sizeof(SConfigObj) + CFG_RESERVE_SIZE; - SSdbRaw *pRaw = sdbAllocRaw(SDB_CFG, CFG_VER_NUMBER, sz); - if (pRaw == NULL) goto _OVER; + SEncoder encoder; + tEncoderInit(&encoder, NULL, 0); + if ((code = tEncodeSConfigObj(&encoder, obj)) < 0) { + tEncoderClear(&encoder); + TSDB_CHECK_CODE(code, lino, _over); + } + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + + int32_t size = sizeof(int32_t) + tlen + CFG_RESERVE_SIZE; + SSdbRaw *pRaw = sdbAllocRaw(SDB_CFG, CFG_VER_NUMBER, size); + TSDB_CHECK_NULL(pRaw, code, lino, _over, terrno); + + buf = taosMemoryMalloc(tlen); + TSDB_CHECK_NULL(buf, code, lino, _over, terrno); + + tEncoderInit(&encoder, buf, tlen); + if ((code = tEncodeSConfigObj(&encoder, obj)) < 0) { + tEncoderClear(&encoder); + TSDB_CHECK_CODE(code, lino, _over); + } + + tEncoderClear(&encoder); int32_t dataPos = 0; - char name[CFG_NAME_MAX_LEN] = {0}; - tstrncpy(name, obj->name, CFG_NAME_MAX_LEN); - SDB_SET_BINARY(pRaw, dataPos, name, CFG_NAME_MAX_LEN, _OVER) - SDB_SET_INT32(pRaw, dataPos, obj->dtype, _OVER) - switch (obj->dtype) { - case CFG_DTYPE_NONE: - break; - case CFG_DTYPE_BOOL: - SDB_SET_BOOL(pRaw, dataPos, obj->bval, _OVER) - break; - case CFG_DTYPE_INT32: - SDB_SET_INT32(pRaw, dataPos, obj->i32, _OVER); - break; - case CFG_DTYPE_INT64: - SDB_SET_INT64(pRaw, dataPos, obj->i64, _OVER); - break; - case CFG_DTYPE_FLOAT: - case CFG_DTYPE_DOUBLE: - SDB_SET_FLOAT(pRaw, dataPos, obj->fval, _OVER) - break; - case CFG_DTYPE_STRING: - case CFG_DTYPE_DIR: - case CFG_DTYPE_LOCALE: - case CFG_DTYPE_CHARSET: - case CFG_DTYPE_TIMEZONE: - SDB_SET_BINARY(pRaw, dataPos, obj->str, TSDB_CONFIG_VALUE_LEN, _OVER) - break; - } - SDB_SET_RESERVE(pRaw, dataPos, CFG_RESERVE_SIZE, _OVER) + SDB_SET_INT32(pRaw, dataPos, tlen, _over); + SDB_SET_BINARY(pRaw, dataPos, buf, tlen, _over); + SDB_SET_DATALEN(pRaw, dataPos, _over); - terrno = 0; - -_OVER: - if (terrno != 0) { - mError("cfg failed to encode to raw:%p since %s", pRaw, terrstr()); +_over: + taosMemoryFreeClear(buf); + if (code != TSDB_CODE_SUCCESS) { + mError("cfg:%s, failed to encode to raw:%p at line:%d since %s", obj->name, pRaw, lino, tstrerror(code)); sdbFreeRaw(pRaw); + terrno = code; return NULL; } - mTrace("cfg encode to raw:%p, row:%p", pRaw, obj); + + terrno = 0; + mTrace("cfg:%s, encode to raw:%p, row:%p", obj->name, pRaw, obj); return pRaw; } SSdbRow *mndCfgActionDecode(SSdbRaw *pRaw) { - int32_t code = 0; - int32_t lino = 0; - int32_t len = -1; - terrno = TSDB_CODE_OUT_OF_MEMORY; + int32_t code = 0; + int32_t lino = 0; SSdbRow *pRow = NULL; - SConfigObj *obj = NULL; + SConfigObj *pObj = NULL; + void *buf = NULL; + int8_t sver = 0; + int32_t tlen; + int32_t dataPos = 0; - int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; + code = sdbGetRawSoftVer(pRaw, &sver); + TSDB_CHECK_CODE(code, lino, _over); if (sver != CFG_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - goto _OVER; + goto _over; } pRow = sdbAllocRow(sizeof(SConfigObj)); - if (pRow == NULL) goto _OVER; + TSDB_CHECK_NULL(pRow, code, lino, _over, terrno); - obj = sdbGetRowObj(pRow); - if (obj == NULL) goto _OVER; - int32_t dataPos = 0; + pObj = sdbGetRowObj(pRow); + TSDB_CHECK_NULL(pObj, code, lino, _over, terrno); - SDB_GET_BINARY(pRaw, dataPos, obj->name, CFG_NAME_MAX_LEN, _OVER) - SDB_GET_INT32(pRaw, dataPos, (int32_t *)&obj->dtype, _OVER) - switch (obj->dtype) { - case CFG_DTYPE_NONE: - break; - case CFG_DTYPE_BOOL: - SDB_GET_BOOL(pRaw, dataPos, &obj->bval, _OVER) - break; - case CFG_DTYPE_INT32: - SDB_GET_INT32(pRaw, dataPos, &obj->i32, _OVER); - break; - case CFG_DTYPE_INT64: - SDB_GET_INT64(pRaw, dataPos, &obj->i64, _OVER); - break; - case CFG_DTYPE_FLOAT: - case CFG_DTYPE_DOUBLE: - SDB_GET_FLOAT(pRaw, dataPos, &obj->fval, _OVER) - break; - case CFG_DTYPE_STRING: - case CFG_DTYPE_DIR: - case CFG_DTYPE_LOCALE: - case CFG_DTYPE_CHARSET: - case CFG_DTYPE_TIMEZONE: - SDB_GET_BINARY(pRaw, dataPos, obj->str, TSDB_CONFIG_VALUE_LEN, _OVER) - break; + SDB_GET_INT32(pRaw, dataPos, &tlen, _over); + + buf = taosMemoryMalloc(tlen + 1); + TSDB_CHECK_NULL(buf, code, lino, _over, terrno); + + SDB_GET_BINARY(pRaw, dataPos, buf, tlen, _over); + + SDecoder decoder; + tDecoderInit(&decoder, buf, tlen + 1); + code = tDecodeSConfigObj(&decoder, pObj); + tDecoderClear(&decoder); + + if (code < 0) { + tFreeSConfigObj(pObj); } - terrno = TSDB_CODE_SUCCESS; -_OVER: - if (terrno != 0) { - mError("cfg failed to decode from raw:%p since %s", pRaw, terrstr()); +_over: + taosMemoryFreeClear(buf); + + if (code != TSDB_CODE_SUCCESS) { + mError("cfg:%s, failed to decode from raw:%p since %s at:%d", pObj->name, pRaw, tstrerror(code), lino); taosMemoryFreeClear(pRow); + terrno = code; return NULL; + } else { + mTrace("config:%s, decode from raw:%p, row:%p", pObj->name, pRaw, pObj); + terrno = 0; + return pRow; } - - mTrace("cfg decode from raw:%p, row:%p", pRaw, obj); - return pRow; } static int32_t mndCfgActionInsert(SSdb *pSdb, SConfigObj *obj) { @@ -211,7 +199,11 @@ static int32_t mndCfgActionUpdate(SSdb *pSdb, SConfigObj *pOld, SConfigObj *pNew case CFG_DTYPE_LOCALE: case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: - tstrncpy(pOld->str, pNew->str, TSDB_CONFIG_VALUE_LEN); + taosMemoryFree(pOld->str); + pOld->str = taosStrdup(pNew->str); + if (pOld->str == NULL) { + return terrno; + } break; } return TSDB_CODE_SUCCESS; @@ -313,11 +305,10 @@ int32_t mndInitWriteCfg(SMnode *pMnode) { SConfigObj *versionObj = mndInitConfigVersion(); if ((code = mndSetCreateConfigCommitLogs(pTrans, versionObj)) != 0) { mError("failed to init mnd config version, since %s", tstrerror(code)); - taosMemoryFree(versionObj->str); - taosMemoryFree(versionObj); + tFreeSConfigObj(versionObj); goto _OVER; } - taosMemoryFree(versionObj); + tFreeSConfigObj(versionObj); sz = taosArrayGetSize(taosGetGlobalCfg(tsCfg)); for (int i = 0; i < sz; ++i) { @@ -329,10 +320,10 @@ int32_t mndInitWriteCfg(SMnode *pMnode) { } if ((code = mndSetCreateConfigCommitLogs(pTrans, obj)) != 0) { mError("failed to init mnd config:%s, since %s", item->name, tstrerror(code)); - taosMemoryFree(obj); + tFreeSConfigObj(obj); goto _OVER; } - taosMemoryFree(obj); + tFreeSConfigObj(obj); } if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER; @@ -412,53 +403,6 @@ int32_t mndSetCreateConfigCommitLogs(STrans *pTrans, SConfigObj *item) { return TSDB_CODE_SUCCESS; } -int32_t cfgUpdateItem(SConfigItem *pItem, SConfigObj *obj) { - int32_t code = TSDB_CODE_SUCCESS; - if (pItem == NULL || obj == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); - } - - switch (pItem->dtype) { - case CFG_DTYPE_BOOL: { - pItem->bval = obj->bval; - break; - } - case CFG_DTYPE_INT32: { - pItem->i32 = obj->i32; - break; - } - case CFG_DTYPE_INT64: { - pItem->i64 = obj->i64; - break; - } - case CFG_DTYPE_FLOAT: - case CFG_DTYPE_DOUBLE: { - pItem->fval = obj->fval; - break; - } - case CFG_DTYPE_DIR: - case CFG_DTYPE_TIMEZONE: - case CFG_DTYPE_CHARSET: - case CFG_DTYPE_LOCALE: - case CFG_DTYPE_NONE: - case CFG_DTYPE_STRING: { - if (obj->str != NULL) { - taosMemoryFree(pItem->str); - pItem->str = taosStrdup(obj->str); - if (pItem->str == NULL) { - TAOS_RETURN(terrno); - } - } - break; - } - default: - code = TSDB_CODE_INVALID_CFG; - break; - } - - TAOS_RETURN(code); -} - static int32_t mndMCfg2DCfg(SMCfgDnodeReq *pMCfgReq, SDCfgDnodeReq *pDCfgReq) { int32_t code = 0; char *p = pMCfgReq->config; @@ -682,26 +626,29 @@ _err: static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue, ECfgDataType dtype, int32_t tsmmConfigVersion) { - int32_t code = -1; - int32_t lino = -1; - SConfigObj pVersion = {0}, pObj = {0}; + int32_t code = -1; + int32_t lino = -1; + SConfigObj *pVersion = taosMemoryMalloc(sizeof(SConfigObj)), *pObj = taosMemoryMalloc(sizeof(SConfigObj)); + if (pVersion == NULL || pObj == NULL) { + code = terrno; + goto _OVER; + } + tstrncpy(pVersion->name, "tsmmConfigVersion", CFG_NAME_MAX_LEN); + pVersion->dtype = CFG_DTYPE_INT32; + pVersion->i32 = tsmmConfigVersion; - tstrncpy(pVersion.name, "tsmmConfigVersion", CFG_NAME_MAX_LEN); - pVersion.i32 = tsmmConfigVersion; - pVersion.dtype = CFG_DTYPE_INT32; + pObj->dtype = dtype; + tstrncpy(pObj->name, name, CFG_NAME_MAX_LEN); - pObj.dtype = dtype; - tstrncpy(pObj.name, name, CFG_NAME_MAX_LEN); - - TAOS_CHECK_GOTO(mndUpdateObj(&pObj, name, pValue), &lino, _OVER); + TAOS_CHECK_GOTO(mndUpdateObj(pObj, name, pValue), &lino, _OVER); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "update-config"); if (pTrans == NULL) { - if (terrno != 0) code = terrno; + code = terrno; goto _OVER; } mInfo("trans:%d, used to update config:%s to value:%s", pTrans->id, name, pValue); - TAOS_CHECK_GOTO(mndSetCreateConfigCommitLogs(pTrans, &pVersion), &lino, _OVER); - TAOS_CHECK_GOTO(mndSetCreateConfigCommitLogs(pTrans, &pObj), &lino, _OVER); + TAOS_CHECK_GOTO(mndSetCreateConfigCommitLogs(pTrans, pVersion), &lino, _OVER); + TAOS_CHECK_GOTO(mndSetCreateConfigCommitLogs(pTrans, pObj), &lino, _OVER); if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER; code = 0; _OVER: @@ -709,6 +656,8 @@ _OVER: mError("failed to update config:%s to value:%s, since %s", name, pValue, tstrerror(code)); } mndTransDrop(pTrans); + tFreeSConfigObj(pVersion); + tFreeSConfigObj(pObj); return code; } @@ -809,7 +758,7 @@ static void cfgObjArrayCleanUp(SArray *array) { int32_t sz = taosArrayGetSize(array); for (int32_t i = 0; i < sz; ++i) { SConfigObj *obj = taosArrayGet(array, i); - taosMemoryFree(obj); + tFreeSConfigObj(obj); } taosArrayDestroy(array); } diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 6d66addd6e..1cc547c585 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -758,7 +758,11 @@ SConfigObj *mndInitConfigObj(SConfigItem *pItem) { case CFG_DTYPE_LOCALE: case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: - tstrncpy(pObj->str, pItem->str, TSDB_CONFIG_VALUE_LEN); + pObj->str = taosStrdup(pItem->str); + if (pObj->str == NULL) { + taosMemoryFree(pObj); + return NULL; + } break; } return pObj; @@ -802,12 +806,15 @@ int32_t mndUpdateObj(SConfigObj *pObjNew, const char *name, char *value) { case CFG_DTYPE_CHARSET: case CFG_DTYPE_LOCALE: case CFG_DTYPE_STRING: { - strncpy(pObjNew->str, value, strlen(value)); - pObjNew->str[strlen(value)] = 0; + pObjNew->str = taosStrdup(value); + if (pObjNew->str == NULL) { + code = terrno; + return code; + } break; } - case CFG_DTYPE_NONE: + break; default: code = TSDB_CODE_INVALID_CFG; break; @@ -820,12 +827,92 @@ SConfigObj *mndInitConfigVersion() { if (pObj == NULL) { return NULL; } - strncpy(pObj->name, "tsmmConfigVersion", CFG_NAME_MAX_LEN); + tstrncpy(pObj->name, "tsmmConfigVersion", CFG_NAME_MAX_LEN); pObj->dtype = CFG_DTYPE_INT32; pObj->i32 = 0; return pObj; } +int32_t tEncodeSConfigObj(SEncoder *pEncoder, const SConfigObj *pObj) { + TAOS_CHECK_RETURN(tStartEncode(pEncoder)); + TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->name)); + + TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pObj->dtype)); + switch (pObj->dtype) { + case CFG_DTYPE_BOOL: + TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->bval)); + break; + case CFG_DTYPE_INT32: + TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pObj->i32)); + break; + case CFG_DTYPE_INT64: + TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->i64)); + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + TAOS_CHECK_RETURN(tEncodeFloat(pEncoder, pObj->fval)); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + if (pObj->str != NULL) { + TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->str)); + } else { + TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, "")); + } + break; + default: + break; + } + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeSConfigObj(SDecoder *pDecoder, SConfigObj *pObj) { + TAOS_CHECK_RETURN(tStartDecode(pDecoder)); + TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pObj->name)); + TAOS_CHECK_RETURN(tDecodeI32(pDecoder, (int32_t *)&pObj->dtype)); + switch (pObj->dtype) { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + TAOS_CHECK_RETURN(tDecodeBool(pDecoder, &pObj->bval)); + break; + case CFG_DTYPE_INT32: + TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pObj->i32)); + break; + case CFG_DTYPE_INT64: + TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->i64)); + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + TAOS_CHECK_RETURN(tDecodeFloat(pDecoder, &pObj->fval)); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + TAOS_CHECK_RETURN(tDecodeCStrAlloc(pDecoder, &pObj->str)); + break; + } + tEndDecode(pDecoder); + TAOS_RETURN(TSDB_CODE_SUCCESS); +} + +void tFreeSConfigObj(SConfigObj *obj) { + if (obj == NULL) { + return; + } + if (obj->dtype == CFG_DTYPE_STRING || obj->dtype == CFG_DTYPE_DIR || obj->dtype == CFG_DTYPE_LOCALE || + obj->dtype == CFG_DTYPE_CHARSET || obj->dtype == CFG_DTYPE_TIMEZONE) { + taosMemoryFreeClear(obj->str); + } + taosMemoryFreeClear(obj); +} + // SMqSubActionLogEntry *tCloneSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) { // SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry)); // if (pEntryNew == NULL) return NULL; From f8a6489bb8401fd4dcd9d44ad75e485afff93008 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 12 Dec 2024 19:31:28 +0800 Subject: [PATCH 92/99] Fix ci problems. --- source/dnode/mnode/impl/src/mndConfig.c | 19 ++++++++++++++----- source/dnode/mnode/impl/src/mndDef.c | 3 +-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index 84a9110bc5..feae65d6a7 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -63,9 +63,10 @@ int32_t mndInitConfig(SMnode *pMnode) { } SSdbRaw *mnCfgActionEncode(SConfigObj *obj) { - int32_t code = 0; - int32_t lino = 0; - void *buf = NULL; + int32_t code = 0; + int32_t lino = 0; + void *buf = NULL; + SSdbRaw *pRaw = NULL; SEncoder encoder; tEncoderInit(&encoder, NULL, 0); @@ -77,8 +78,8 @@ SSdbRaw *mnCfgActionEncode(SConfigObj *obj) { int32_t tlen = encoder.pos; tEncoderClear(&encoder); - int32_t size = sizeof(int32_t) + tlen + CFG_RESERVE_SIZE; - SSdbRaw *pRaw = sdbAllocRaw(SDB_CFG, CFG_VER_NUMBER, size); + int32_t size = sizeof(int32_t) + tlen; + pRaw = sdbAllocRaw(SDB_CFG, CFG_VER_NUMBER, size); TSDB_CHECK_NULL(pRaw, code, lino, _over, terrno); buf = taosMemoryMalloc(tlen); @@ -173,6 +174,7 @@ static int32_t mndCfgActionInsert(SSdb *pSdb, SConfigObj *obj) { static int32_t mndCfgActionDelete(SSdb *pSdb, SConfigObj *obj) { mTrace("cfg:%s, perform delete action, row:%p", obj->name, obj); + tFreeSConfigObj(obj); return 0; } @@ -306,9 +308,11 @@ int32_t mndInitWriteCfg(SMnode *pMnode) { if ((code = mndSetCreateConfigCommitLogs(pTrans, versionObj)) != 0) { mError("failed to init mnd config version, since %s", tstrerror(code)); tFreeSConfigObj(versionObj); + taosMemoryFree(versionObj); goto _OVER; } tFreeSConfigObj(versionObj); + taosMemoryFree(versionObj); sz = taosArrayGetSize(taosGetGlobalCfg(tsCfg)); for (int i = 0; i < sz; ++i) { @@ -321,9 +325,11 @@ int32_t mndInitWriteCfg(SMnode *pMnode) { if ((code = mndSetCreateConfigCommitLogs(pTrans, obj)) != 0) { mError("failed to init mnd config:%s, since %s", item->name, tstrerror(code)); tFreeSConfigObj(obj); + taosMemoryFree(obj); goto _OVER; } tFreeSConfigObj(obj); + taosMemoryFree(obj); } if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER; @@ -657,7 +663,9 @@ _OVER: } mndTransDrop(pTrans); tFreeSConfigObj(pVersion); + taosMemoryFree(pVersion); tFreeSConfigObj(pObj); + taosMemoryFree(pObj); return code; } @@ -759,6 +767,7 @@ static void cfgObjArrayCleanUp(SArray *array) { for (int32_t i = 0; i < sz; ++i) { SConfigObj *obj = taosArrayGet(array, i); tFreeSConfigObj(obj); + taosMemoryFree(obj); } taosArrayDestroy(array); } diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 1cc547c585..33ea98d906 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -908,9 +908,8 @@ void tFreeSConfigObj(SConfigObj *obj) { } if (obj->dtype == CFG_DTYPE_STRING || obj->dtype == CFG_DTYPE_DIR || obj->dtype == CFG_DTYPE_LOCALE || obj->dtype == CFG_DTYPE_CHARSET || obj->dtype == CFG_DTYPE_TIMEZONE) { - taosMemoryFreeClear(obj->str); + taosMemoryFree(obj->str); } - taosMemoryFreeClear(obj); } // SMqSubActionLogEntry *tCloneSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) { From 19ea0423185196b2b8e42397de597d0acea9a18b Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Fri, 13 Dec 2024 12:32:08 +0800 Subject: [PATCH 93/99] Fix some review errors. --- source/dnode/mnode/impl/src/mndConfig.c | 2 -- source/dnode/mnode/sdb/src/sdbFile.c | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index feae65d6a7..06555d47a6 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -218,7 +218,6 @@ static int32_t mndCfgActionPrepare(SMnode *pMnode) { return mndTryRebuildCfg(pMn static int32_t mndProcessConfigReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SConfigReq configReq = {0}; - SDnodeObj *pDnode = NULL; int32_t code = TSDB_CODE_SUCCESS; SArray *array = NULL; @@ -288,7 +287,6 @@ _OVER: } sdbRelease(pMnode->pSdb, vObj); cfgArrayCleanUp(array); - mndReleaseDnode(pMnode, pDnode); return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 156938ed22..2d752a2aff 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -53,7 +53,7 @@ static int32_t sdbPrepareData(SSdb *pSdb) { mInfo("start to prepare sdb"); for (int32_t i = SDB_MAX - 1; i >= 0; --i) { - SdbDeployFp fp = pSdb->prepareFps[i]; + SdbPrepareFp fp = pSdb->prepareFps[i]; if (fp == NULL) continue; mInfo("start to prepare sdb:%s", sdbTableName(i)); From c8bc3b391d1d39bd45f17e5b257b34f7d0998aa8 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Fri, 13 Dec 2024 14:05:44 +0800 Subject: [PATCH 94/99] Set tsDiskIDCheckEnabled default = false. --- source/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 98e19b5195..f620db240a 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -288,7 +288,7 @@ int32_t tsTtlFlushThreshold = 100; /* maximum number of dirty items in memory. int32_t tsTtlBatchDropNum = 10000; // number of tables dropped per batch // internal -bool tsDiskIDCheckEnabled = true; +bool tsDiskIDCheckEnabled = false; int32_t tsTransPullupInterval = 2; int32_t tsCompactPullupInterval = 10; int32_t tsMqRebalanceInterval = 2; From e20b6c1188b5c629a0fb31e09f4b993defd01493 Mon Sep 17 00:00:00 2001 From: Pengrongkun Date: Fri, 13 Dec 2024 14:38:19 +0800 Subject: [PATCH 95/99] fix TD-33167 --- source/client/src/clientMain.c | 14 +- source/client/src/clientStmt2.c | 12 +- tests/script/api/stmt2-performance.c | 242 ++++++++++++++++----------- 3 files changed, 157 insertions(+), 111 deletions(-) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 3d22c26fa7..f4747587bf 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -2175,13 +2175,6 @@ int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col } } - if (bindv->tags && bindv->tags[i]) { - code = stmtSetTbTags2(stmt, bindv->tags[i]); - if (code) { - return code; - } - } - if (bindv->bind_cols && bindv->bind_cols[i]) { TAOS_STMT2_BIND *bind = bindv->bind_cols[i]; @@ -2204,6 +2197,13 @@ int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col return code; } } + + if (bindv->tags && bindv->tags[i]) { + code = stmtSetTbTags2(stmt, bindv->tags[i]); + if (code) { + return code; + } + } } return TSDB_CODE_SUCCESS; diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index 2f68b99c81..ec6530d871 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -109,7 +109,7 @@ static int32_t stmtSwitchStatus(STscStmt2* pStmt, STMT_STATUS newStatus) { } break; case STMT_SETTAGS: - if (STMT_STATUS_NE(SETTBNAME) && STMT_STATUS_NE(FETCH_FIELDS)) { + if (STMT_STATUS_NE(SETTBNAME) && STMT_STATUS_NE(FETCH_FIELDS) && STMT_STATUS_NE(ADD_BATCH)) { code = TSDB_CODE_TSC_STMT_API_ERROR; } break; @@ -145,7 +145,7 @@ static int32_t stmtSwitchStatus(STscStmt2* pStmt, STMT_STATUS newStatus) { code = TSDB_CODE_TSC_STMT_API_ERROR; } } else { - if (STMT_STATUS_NE(ADD_BATCH) && STMT_STATUS_NE(FETCH_FIELDS)) { + if (STMT_STATUS_NE(ADD_BATCH) && STMT_STATUS_NE(FETCH_FIELDS) && STMT_STATUS_NE(SETTAGS)) { code = TSDB_CODE_TSC_STMT_API_ERROR; } } @@ -952,6 +952,14 @@ int stmtSetTbName2(TAOS_STMT2* stmt, const char* tbName) { if (!pStmt->sql.stbInterlaceMode || NULL == pStmt->sql.siInfo.pDataCtx) { STMT_ERR_RET(stmtCreateRequest(pStmt)); + if (pStmt->exec.pRequest->pDb == NULL) { + char* dbName = NULL; + if (qParseDbName(pStmt->exec.pRequest->sqlstr, pStmt->exec.pRequest->sqlLen, &dbName)) { + STMT_ERR_RET(stmtSetDbName2(pStmt, dbName)); + taosMemoryFreeClear(dbName); + } + } + STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); STMT_ERR_RET(tNameExtractFullName(&pStmt->bInfo.sname, pStmt->bInfo.tbFName)); diff --git a/tests/script/api/stmt2-performance.c b/tests/script/api/stmt2-performance.c index a0ab7f5e0d..aa8e5b9450 100644 --- a/tests/script/api/stmt2-performance.c +++ b/tests/script/api/stmt2-performance.c @@ -5,8 +5,9 @@ #include #include "taos.h" -int CTB_NUMS = 10; -int ROW_NUMS = 2; +int CTB_NUMS = 1000; +int ROW_NUMS = 10; +int CYC_NUMS = 5; void do_query(TAOS* taos, const char* sql) { TAOS_RES* result = taos_query(taos, sql); @@ -19,10 +20,32 @@ void do_query(TAOS* taos, const char* sql) { taos_free_result(result); } -void do_stmt(TAOS* taos, const char* sql) { +void do_ins(TAOS* taos, const char* sql) { + TAOS_RES* result = taos_query(taos, sql); + int code = taos_errno(result); + if (code) { + printf("failed to query: %s, reason:%s\n", sql, taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); +} + +void createCtb(TAOS* taos, const char* tbname) { + char* tmp = (char*)malloc(sizeof(char) * 100); + sprintf(tmp, "create table db.%s using db.stb tags(0, 'after')", tbname); + do_query(taos, tmp); +} + +void initEnv(TAOS* taos) { do_query(taos, "drop database if exists db"); do_query(taos, "create database db"); do_query(taos, "create table db.stb (ts timestamp, b binary(10)) tags(t1 int, t2 binary(10))"); + do_query(taos, "use db"); +} + +void do_stmt(TAOS* taos, const char* sql) { + initEnv(taos); TAOS_STMT2_OPTION option = {0, true, true, NULL, NULL}; @@ -35,35 +58,107 @@ void do_stmt(TAOS* taos, const char* sql) { } int fieldNum = 0; TAOS_FIELD_STB* pFields = NULL; - code = taos_stmt2_get_stb_fields(stmt, &fieldNum, &pFields); - if (code != 0) { - printf("failed get col,ErrCode: 0x%x, ErrMessage: %s.\n", code, taos_stmt2_error(stmt)); - } else { - printf("col nums:%d\n", fieldNum); - for (int i = 0; i < fieldNum; i++) { - printf("field[%d]: %s, data_type:%d, field_type:%d\n", i, pFields[i].name, pFields[i].type, - pFields[i].field_type); - } - } + // code = taos_stmt2_get_stb_fields(stmt, &fieldNum, &pFields); + // if (code != 0) { + // printf("failed get col,ErrCode: 0x%x, ErrMessage: %s.\n", code, taos_stmt2_error(stmt)); + // } else { + // printf("col nums:%d\n", fieldNum); + // for (int i = 0; i < fieldNum; i++) { + // printf("field[%d]: %s, data_type:%d, field_type:%d\n", i, pFields[i].name, pFields[i].type, + // pFields[i].field_type); + // } + // } // tbname char** tbs = (char**)malloc(CTB_NUMS * sizeof(char*)); for (int i = 0; i < CTB_NUMS; i++) { tbs[i] = (char*)malloc(sizeof(char) * 20); sprintf(tbs[i], "ctb_%d", i); - // char* tmp=(char*)malloc(sizeof(char) * 100); - // sprintf(tmp, "create table db.%s using db.stb tags(0, 'abc')", tbs[i]); - // do_query(taos, tmp); + createCtb(taos, tbs[i]); } - // col params - int64_t** ts = (int64_t**)malloc(CTB_NUMS * sizeof(int64_t*)); + for (int r = 0; r < CYC_NUMS; r++) { + // col params + int64_t** ts = (int64_t**)malloc(CTB_NUMS * sizeof(int64_t*)); + char** b = (char**)malloc(CTB_NUMS * sizeof(char*)); + int* ts_len = (int*)malloc(ROW_NUMS * sizeof(int)); + int* b_len = (int*)malloc(ROW_NUMS * sizeof(int)); + for (int i = 0; i < ROW_NUMS; i++) { + ts_len[i] = sizeof(int64_t); + b_len[i] = 1; + } + for (int i = 0; i < CTB_NUMS; i++) { + ts[i] = (int64_t*)malloc(ROW_NUMS * sizeof(int64_t)); + b[i] = (char*)malloc(ROW_NUMS * sizeof(char)); + for (int j = 0; j < ROW_NUMS; j++) { + ts[i][j] = 1591060628000 + r * 100000 + j; + b[i][j] = 'a' + j; + } + } + // tag params + int t1 = 0; + int t1len = sizeof(int); + int t2len = 3; + // TAOS_STMT2_BIND* tagv[2] = {&tags[0][0], &tags[1][0]}; + + clock_t start, end; + double cpu_time_used; + + // bind params + TAOS_STMT2_BIND** paramv = (TAOS_STMT2_BIND**)malloc(CTB_NUMS * sizeof(TAOS_STMT2_BIND*)); + TAOS_STMT2_BIND** tags = (TAOS_STMT2_BIND**)malloc(CTB_NUMS * sizeof(TAOS_STMT2_BIND*)); + for (int i = 0; i < CTB_NUMS; i++) { + // create tags + tags[i] = (TAOS_STMT2_BIND*)malloc(2 * sizeof(TAOS_STMT2_BIND)); + tags[i][0] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_INT, &t1, &t1len, NULL, 0}; + tags[i][1] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_BINARY, "after", &t2len, NULL, 0}; + + // create col params + paramv[i] = (TAOS_STMT2_BIND*)malloc(2 * sizeof(TAOS_STMT2_BIND)); + paramv[i][0] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_TIMESTAMP, &ts[i][0], &ts_len[0], NULL, ROW_NUMS}; + paramv[i][1] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_BINARY, &b[i][0], &b_len[0], NULL, ROW_NUMS}; + } + // bind + start = clock(); + TAOS_STMT2_BINDV bindv = {CTB_NUMS, tbs, tags, paramv}; + if (taos_stmt2_bind_param(stmt, &bindv, -1)) { + printf("failed to execute taos_stmt2_bind_param statement.error:%s\n", taos_stmt2_error(stmt)); + taos_stmt2_close(stmt); + return; + } + end = clock(); + cpu_time_used = ((double)(end - start)) / CLOCKS_PER_SEC; + printf("stmt2-bind [%s] insert Time used: %f seconds\n", sql, cpu_time_used); + start = clock(); + // exec + if (taos_stmt2_exec(stmt, NULL)) { + printf("failed to execute taos_stmt2_exec statement.error:%s\n", taos_stmt2_error(stmt)); + taos_stmt2_close(stmt); + return; + } + end = clock(); + cpu_time_used = ((double)(end - start)) / CLOCKS_PER_SEC; + printf("stmt2-exec [%s] insert Time used: %f seconds\n", sql, cpu_time_used); + } + + // taos_stmt2_free_fields(stmt, pFields); + taos_stmt2_close(stmt); +} + +void do_taosc(TAOS* taos) { + initEnv(taos); + // cols + int64_t** ts = (int64_t*)malloc(CTB_NUMS * sizeof(int64_t)); char** b = (char**)malloc(CTB_NUMS * sizeof(char*)); - int* ts_len = (int*)malloc(ROW_NUMS * sizeof(int)); - int* b_len = (int*)malloc(ROW_NUMS * sizeof(int)); - for (int i = 0; i < ROW_NUMS; i++) { - ts_len[i] = sizeof(int64_t); - b_len[i] = 1; + // ctbnames + char** tbs = (char**)malloc(CTB_NUMS * sizeof(char*)); + + // create table before insert + for (int i = 0; i < CTB_NUMS; i++) { + tbs[i] = (char*)malloc(sizeof(char) * 20); + sprintf(tbs[i], "ctb_%d", i); + createCtb(taos, tbs[i]); } + for (int i = 0; i < CTB_NUMS; i++) { ts[i] = (int64_t*)malloc(ROW_NUMS * sizeof(int64_t)); b[i] = (char*)malloc(ROW_NUMS * sizeof(char)); @@ -72,87 +167,30 @@ void do_stmt(TAOS* taos, const char* sql) { b[i][j] = 'a' + j; } } - // tag params - int t1 = 0; - int t1len = sizeof(int); - int t2len = 3; - // TAOS_STMT2_BIND* tagv[2] = {&tags[0][0], &tags[1][0]}; + for (int r = 0; r < CYC_NUMS; r++) { + clock_t start, end; + double cpu_time_used; + char* tsc_sql = malloc(sizeof(char) * 1000000); + sprintf(tsc_sql, "insert into db.stb(tbname,ts,b,t1,t2) values"); - clock_t start, end; - double cpu_time_used; - - // bind params - TAOS_STMT2_BIND** paramv = (TAOS_STMT2_BIND**)malloc(CTB_NUMS * sizeof(TAOS_STMT2_BIND*)); - TAOS_STMT2_BIND** tags = (TAOS_STMT2_BIND**)malloc(CTB_NUMS * sizeof(TAOS_STMT2_BIND*)); - for (int i = 0; i < CTB_NUMS; i++) { - // create tags - tags[i] = (TAOS_STMT2_BIND*)malloc(2 * sizeof(TAOS_STMT2_BIND)); - tags[i][0] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_INT, &t1, &t1len, NULL, 0}; - tags[i][1] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_BINARY, "after", &t2len, NULL, 0}; - - // create col params - paramv[i] = (TAOS_STMT2_BIND*)malloc(2 * sizeof(TAOS_STMT2_BIND)); - paramv[i][0] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_TIMESTAMP, &ts[i][0], &ts_len[0], NULL, ROW_NUMS}; - paramv[i][1] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_BINARY, &b[i][0], &b_len[0], NULL, ROW_NUMS}; - } - // bind - start = clock(); - TAOS_STMT2_BINDV bindv = {CTB_NUMS, tbs, tags, paramv}; - if (taos_stmt2_bind_param(stmt, &bindv, -1)) { - printf("failed to execute taos_stmt2_bind_param statement.error:%s\n", taos_stmt2_error(stmt)); - taos_stmt2_close(stmt); - return; - } - end = clock(); - cpu_time_used = ((double)(end - start)) / CLOCKS_PER_SEC; - printf("stmt2-bind [%s] insert Time used: %f seconds\n", sql, cpu_time_used); - start = clock(); - // exec - if (taos_stmt2_exec(stmt, NULL)) { - printf("failed to execute taos_stmt2_exec statement.error:%s\n", taos_stmt2_error(stmt)); - taos_stmt2_close(stmt); - return; - } - end = clock(); - cpu_time_used = ((double)(end - start)) / CLOCKS_PER_SEC; - printf("stmt2-exec [%s] insert Time used: %f seconds\n", sql, cpu_time_used); - - taos_stmt2_free_fields(stmt, pFields); - taos_stmt2_close(stmt); -} - -void do_taosc(TAOS* taos) { - do_query(taos, "drop database if exists testdb"); - do_query(taos, "create database testdb"); - do_query(taos, "create table testdb.stb (ts timestamp, b binary(10)) tags(t1 int, t2 binary(10))"); - int64_t* ts = (int64_t*)malloc(CTB_NUMS * sizeof(int64_t)); - char** tbs = (char**)malloc(CTB_NUMS * sizeof(char*)); - for (int i = 0; i < CTB_NUMS; i++) { - tbs[i] = (char*)malloc(sizeof(char) * 10); - } - clock_t start, end; - double cpu_time_used; - char* tsc_sql = malloc(sizeof(char) * 1000000); - sprintf(tsc_sql, "insert into testdb.stb(tbname,ts,b,t1,t2) values"); - - for (int j = 0; j < ROW_NUMS; j++) { for (int i = 0; i < CTB_NUMS; i++) { - ts[i] = 1591060628000 + i + j * CTB_NUMS; - snprintf(tbs[i], sizeof(tbs[i]), "ctb_%d", i + j * CTB_NUMS); - if (i == CTB_NUMS - 1 && j == ROW_NUMS - 1) { - sprintf(tsc_sql + strlen(tsc_sql), "('%s',%lld,'abc',0,'after')", tbs[i], ts[i]); - } else { - sprintf(tsc_sql + strlen(tsc_sql), "('%s',%lld,'abc'0,'after'),", tbs[i], ts[i]); + snprintf(tbs[i], sizeof(tbs[i]), "ctb_%d", i); + for (int j = 0; j < ROW_NUMS; j++) { + if (i == CTB_NUMS - 1 && j == ROW_NUMS - 1) { + sprintf(tsc_sql + strlen(tsc_sql), "('%s',%lld,'%c',0,'after')", tbs[i], ts[i][j] + r * 10000, b[i][j]); + } else { + sprintf(tsc_sql + strlen(tsc_sql), "('%s',%lld,'%c',0,'after'),", tbs[i], ts[i][j] + r * 10000, b[i][j]); + } } } - } - start = clock(); - // printf("%s", tsc_sql); - taos_query(taos, tsc_sql); - end = clock(); - cpu_time_used = ((double)(end - start)) / CLOCKS_PER_SEC; - printf("taosc insert Time used: %f seconds\n", cpu_time_used); + start = clock(); + // printf("%s", tsc_sql); + do_ins(taos, tsc_sql); + end = clock(); + cpu_time_used = ((double)(end - start)) / CLOCKS_PER_SEC; + printf("taosc insert Time used: %f seconds\n", cpu_time_used); + } } int main() { @@ -162,10 +200,10 @@ int main() { exit(1); } - // sleep(3); + sleep(3); do_stmt(taos, "insert into db.stb(tbname,ts,b,t1,t2) values(?,?,?,?,?)"); - do_stmt(taos, "insert into db.? using db.stb tags(?,?)values(?,?)"); - // do_taosc(taos); + // do_stmt(taos, "insert into db.? using db.stb tags(?,?)values(?,?)"); + do_taosc(taos); taos_close(taos); taos_cleanup(); } From c8cbcc4a3b51371e1dea94e4135b46cc59a48a82 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Fri, 13 Dec 2024 15:52:38 +0800 Subject: [PATCH 96/99] Merge lost file. --- source/util/src/tconfig.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index cf9bfe525f..8ede489523 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -747,17 +747,17 @@ int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scop } int32_t cfgAddInt32Ex(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, - int8_t dynScope) { + int8_t dynScope, int8_t category) { SConfigItem item = {.dtype = CFG_DTYPE_INT32, .i32 = defaultVal, .imin = minval, .imax = maxval, .scope = scope, - .dynScope = dynScope}; + .dynScope = dynScope, + .category = category}; return cfgAddItem(pCfg, &item, name); } - int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, int8_t dynScope, int8_t category) { if (defaultVal < minval || defaultVal > maxval) { From 0ac61ea9e2c664adaad94353c303046bddf43994 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Sat, 14 Dec 2024 00:29:01 +0800 Subject: [PATCH 97/99] Fix ci problems. --- source/util/src/tconfig.c | 2 +- tests/system-test/0-others/qmemCtrl.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 8ede489523..109bb16f51 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -557,7 +557,7 @@ int32_t cfgSetItemVal(SConfigItem *pItem, const char *name, const char *value, E break; } - TAOS_RETURN(TSDB_CODE_SUCCESS); + TAOS_RETURN(code); } SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName) { diff --git a/tests/system-test/0-others/qmemCtrl.py b/tests/system-test/0-others/qmemCtrl.py index 2aacc2c6ed..a2eedbec0e 100644 --- a/tests/system-test/0-others/qmemCtrl.py +++ b/tests/system-test/0-others/qmemCtrl.py @@ -19,7 +19,7 @@ from util.common import * from util.sqlset import * class TDTestCase: - updatecfgDict = {'queryUseMemoryPool':'true','minReservedMemorySize':1025, 'singleQueryMaxMemorySize': 0} + updatecfgDict = {'forceReadConfig':'1','queryUseMemoryPool':'true','minReservedMemorySize':1025, 'singleQueryMaxMemorySize': 0} def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) From 997af901f9e77c6e2fa132d80875de1212cb82b7 Mon Sep 17 00:00:00 2001 From: Pengrongkun Date: Fri, 13 Dec 2024 23:43:20 +0800 Subject: [PATCH 98/99] fix review --- source/client/inc/clientStmt2.h | 1 + source/client/src/clientMain.c | 14 +++++++------- source/client/src/clientStmt2.c | 31 +++++++++++++++++++++---------- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/source/client/inc/clientStmt2.h b/source/client/inc/clientStmt2.h index 64abf31bc1..01cf76965a 100644 --- a/source/client/inc/clientStmt2.h +++ b/source/client/inc/clientStmt2.h @@ -150,6 +150,7 @@ typedef struct { SStmtExecInfo exec; SStmtBindInfo bInfo; + char *db; int64_t reqid; int32_t errCode; tsem_t asyncQuerySem; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index f4747587bf..3d22c26fa7 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -2175,6 +2175,13 @@ int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col } } + if (bindv->tags && bindv->tags[i]) { + code = stmtSetTbTags2(stmt, bindv->tags[i]); + if (code) { + return code; + } + } + if (bindv->bind_cols && bindv->bind_cols[i]) { TAOS_STMT2_BIND *bind = bindv->bind_cols[i]; @@ -2197,13 +2204,6 @@ int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col return code; } } - - if (bindv->tags && bindv->tags[i]) { - code = stmtSetTbTags2(stmt, bindv->tags[i]); - if (code) { - return code; - } - } } return TSDB_CODE_SUCCESS; diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index ec6530d871..788f9b4e29 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -75,6 +75,10 @@ static int32_t stmtCreateRequest(STscStmt2* pStmt) { if (pStmt->reqid != 0) { pStmt->reqid++; } + if (pStmt->db != NULL) { + taosMemoryFreeClear(pStmt->exec.pRequest->pDb); + pStmt->exec.pRequest->pDb = strdup(pStmt->db); + } if (TSDB_CODE_SUCCESS == code) { pStmt->exec.pRequest->syncQuery = true; pStmt->exec.pRequest->isStmtBind = true; @@ -109,7 +113,7 @@ static int32_t stmtSwitchStatus(STscStmt2* pStmt, STMT_STATUS newStatus) { } break; case STMT_SETTAGS: - if (STMT_STATUS_NE(SETTBNAME) && STMT_STATUS_NE(FETCH_FIELDS) && STMT_STATUS_NE(ADD_BATCH)) { + if (STMT_STATUS_EQ(INIT)) { code = TSDB_CODE_TSC_STMT_API_ERROR; } break; @@ -145,7 +149,7 @@ static int32_t stmtSwitchStatus(STscStmt2* pStmt, STMT_STATUS newStatus) { code = TSDB_CODE_TSC_STMT_API_ERROR; } } else { - if (STMT_STATUS_NE(ADD_BATCH) && STMT_STATUS_NE(FETCH_FIELDS) && STMT_STATUS_NE(SETTAGS)) { + if (STMT_STATUS_NE(ADD_BATCH) && STMT_STATUS_NE(FETCH_FIELDS)) { code = TSDB_CODE_TSC_STMT_API_ERROR; } } @@ -412,6 +416,7 @@ static void stmtFreeTbCols(void* buf) { static int32_t stmtCleanSQLInfo(STscStmt2* pStmt) { STMT_DLOG_E("start to free SQL info"); + taosMemoryFreeClear(pStmt->db); taosMemoryFree(pStmt->sql.pBindInfo); taosMemoryFree(pStmt->sql.queryRes.fields); taosMemoryFree(pStmt->sql.queryRes.userFields); @@ -842,6 +847,7 @@ static int stmtSetDbName2(TAOS_STMT2* stmt, const char* dbName) { STMT_DLOG("start to set dbName: %s", dbName); + pStmt->db = taosStrdup(dbName); STMT_ERR_RET(stmtCreateRequest(pStmt)); // The SQL statement specifies a database name, overriding the previously specified database @@ -952,14 +958,6 @@ int stmtSetTbName2(TAOS_STMT2* stmt, const char* tbName) { if (!pStmt->sql.stbInterlaceMode || NULL == pStmt->sql.siInfo.pDataCtx) { STMT_ERR_RET(stmtCreateRequest(pStmt)); - if (pStmt->exec.pRequest->pDb == NULL) { - char* dbName = NULL; - if (qParseDbName(pStmt->exec.pRequest->sqlstr, pStmt->exec.pRequest->sqlLen, &dbName)) { - STMT_ERR_RET(stmtSetDbName2(pStmt, dbName)); - taosMemoryFreeClear(dbName); - } - } - STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); STMT_ERR_RET(tNameExtractFullName(&pStmt->bInfo.sname, pStmt->bInfo.tbFName)); @@ -1000,6 +998,19 @@ int stmtSetTbTags2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* tags) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS)); + if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && + STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { + pStmt->bInfo.needParse = false; + } + STMT_ERR_RET(stmtCreateRequest(pStmt)); + + if (pStmt->bInfo.needParse) { + STMT_ERR_RET(stmtParseSql(pStmt)); + } + if (pStmt->sql.stbInterlaceMode && NULL == pStmt->sql.siInfo.pDataCtx) { + STMT_ERR_RET(stmtInitStbInterlaceTableInfo(pStmt)); + } + SBoundColInfo* tags_info = (SBoundColInfo*)pStmt->bInfo.boundTags; if (tags_info->numOfBound <= 0 || tags_info->numOfCols <= 0) { tscWarn("no tags or cols bound in sql, will not bound tags"); From bdb96ee86d4ee2a025e7b8bcfa7639e4dcefe739 Mon Sep 17 00:00:00 2001 From: menshibin Date: Sat, 14 Dec 2024 14:24:53 +0800 Subject: [PATCH 99/99] modify node.js tmq case --- docs/examples/node/websocketexample/sql_example.js | 1 - docs/examples/node/websocketexample/tmq_example.js | 1 - docs/examples/node/websocketexample/tmq_seek_example.js | 4 ++-- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/examples/node/websocketexample/sql_example.js b/docs/examples/node/websocketexample/sql_example.js index c120b84d99..7e78bb6c64 100644 --- a/docs/examples/node/websocketexample/sql_example.js +++ b/docs/examples/node/websocketexample/sql_example.js @@ -26,7 +26,6 @@ async function createDbAndTable() { let conf = new taos.WSConfig(dsn); conf.setUser('root'); conf.setPwd('taosdata'); - conf.setDb('power'); wsSql = await taos.sqlConnect(conf); console.log("Connected to " + dsn + " successfully."); // create database diff --git a/docs/examples/node/websocketexample/tmq_example.js b/docs/examples/node/websocketexample/tmq_example.js index 4f7f099c31..4ea3db55c3 100644 --- a/docs/examples/node/websocketexample/tmq_example.js +++ b/docs/examples/node/websocketexample/tmq_example.js @@ -40,7 +40,6 @@ async function prepare() { let conf = new taos.WSConfig('ws://localhost:6041'); conf.setUser('root'); conf.setPwd('taosdata'); - conf.setDb('power'); const createDB = `CREATE DATABASE IF NOT EXISTS ${db}`; const createStable = `CREATE STABLE IF NOT EXISTS ${db}.${stable} (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);`; diff --git a/docs/examples/node/websocketexample/tmq_seek_example.js b/docs/examples/node/websocketexample/tmq_seek_example.js index be4d8ddfa4..67286f34b1 100644 --- a/docs/examples/node/websocketexample/tmq_seek_example.js +++ b/docs/examples/node/websocketexample/tmq_seek_example.js @@ -34,10 +34,10 @@ async function createConsumer() { } async function prepare() { - let conf = new taos.WSConfig('ws://192.168.1.98:6041'); + let conf = new taos.WSConfig('ws://localhost:6041'); conf.setUser('root'); conf.setPwd('taosdata'); - conf.setDb('power'); + const createDB = `CREATE DATABASE IF NOT EXISTS ${db}`; const createStable = `CREATE STABLE IF NOT EXISTS ${db}.${stable} (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);`;