From db67245af0f0775f7eb51082d1b5702cd8ade0f4 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Fri, 22 Nov 2024 15:57:18 +0800 Subject: [PATCH] Add more dyn configs and fix vesion in mnd bug. --- include/util/tconfig.h | 4 +- source/common/src/tglobal.c | 58 +++++++++++++++- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- source/dnode/mnode/impl/src/mndConfig.c | 24 ++++--- source/libs/command/src/command.c | 6 +- source/util/src/tconfig.c | 75 ++++++++++++++------- 6 files changed, 129 insertions(+), 40 deletions(-) diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 4ba9a9c93e..cfccbc16c8 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -118,9 +118,9 @@ void cfgCleanup(SConfig *pCfg); int32_t cfgGetSize(SConfig *pCfg); SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName); int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype, bool lock); -int32_t cfgGetAndSetItem(SConfig *pCfg, SConfigItem *pItem, const char *name, const char *value, ECfgSrcType stype, +int32_t cfgGetAndSetItem(SConfig *pCfg, SConfigItem **ppItem, const char *name, const char *value, ECfgSrcType stype, bool lock); -int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer); +int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer, bool isUpdateAll); int32_t cfgCreateIter(SConfig *pConf, SConfigIter **ppIter); SConfigItem *cfgNextIter(SConfigIter *pIter); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 7cd05f7fb6..a45d3b59dc 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -45,7 +45,7 @@ uint16_t tsServerPort = 6030; int32_t tsVersion = 30000000; int32_t tsForceReadConfig = 0; int32_t tsdmConfigVersion = -1; -int32_t tsmmConfigVersion = 0; +int32_t tsmmConfigVersion = -1; int32_t tsConfigInited = 0; int32_t tsStatusInterval = 1; // second int32_t tsNumOfSupportVnodes = 256; @@ -2225,12 +2225,33 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"asynclog", &tsAsyncLog}, {"disableStream", &tsDisableStream}, {"enableWhiteList", &tsEnableWhiteList}, + {"statusInterval", &tsStatusInterval}, {"telemetryReporting", &tsEnableTelem}, {"monitor", &tsEnableMonitor}, {"monitorInterval", &tsMonitorInterval}, + {"monitorComp", &tsMonitorComp}, + {"monitorForceV2", &tsMonitorForceV2}, + {"monitorLogProtocol", &tsMonitorLogProtocol}, + {"monitorMaxLogs", &tsMonitorMaxLogs}, + {"audit", &tsEnableAudit}, + {"auditCreateTable", &tsEnableAuditCreateTable}, + {"auditInterval", &tsAuditInterval}, {"slowLogThreshold", &tsSlowLogThreshold}, + {"compressMsgSize", &tsCompressMsgSize}, + {"compressor", &tsCompressor}, + {"curRange", &tsCurRange}, + {"dPrecision", &tsDPrecision}, + {"fPrecision", &tsFPrecision}, + {"ifAdtFse", &tsIfAdtFse}, + {"maxRange", &tsMaxRange}, + {"maxTsmaNum", &tsMaxTsmaNum}, + {"queryRsmaTolerance", &tsQueryRsmaTolerance}, + {"countAlwaysReturnValue", &tsCountAlwaysReturnValue}, + {"uptimeInterval", &tsUptimeInterval}, {"slowLogThresholdTest", &tsSlowLogThresholdTest}, {"slowLogMaxLen", &tsSlowLogMaxLen}, + {"slowLogScope", &tsSlowLogScope}, + {"slowLogExceptDb", &tsSlowLogExceptDb}, {"mndSdbWriteDelta", &tsMndSdbWriteDelta}, {"minDiskFreeSize", &tsMinDiskFreeSize}, @@ -2238,9 +2259,28 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"randErrorDivisor", &tsRandErrDivisor}, {"randErrorScope", &tsRandErrScope}, {"syncLogBufferMemoryAllowed", &tsLogBufferMemoryAllowed}, + {"resolveFQDNRetryTime", &tsResolveFQDNRetryTime}, + {"syncElectInterval", &tsElectInterval}, + {"syncHeartbeatInterval", &tsHeartbeatInterval}, + {"syncHeartbeatTimeout", &tsHeartbeatTimeout}, + {"syncSnapReplMaxWaitN", &tsSnapReplMaxWaitN}, + {"walFsyncDataSizeLimit", &tsWalFsyncDataSizeLimit}, + + {"numOfCores", &tsNumOfCores}, + + {"enableCoreFile", &tsEnableCoreFile}, + + {"telemetryInterval", &tsTelemInterval}, + {"telemetryPort", &tsTelemPort}, {"cacheLazyLoadThreshold", &tsCacheLazyLoadThreshold}, {"checkpointInterval", &tsStreamCheckpointInterval}, + {"concurrentCheckpoint", &tsMaxConcurrentCheckpoint}, + + {"retentionSpeedLimitMB", &tsRetentionSpeedLimitMB}, + {"trimVDbIntervalSec", &tsTrimVDbIntervalSec}, + {"ttlChangeOnWrite", &tsTtlChangeOnWrite}, + {"keepAliveIdle", &tsKeepAliveIdle}, {"logKeepDays", &tsLogKeepDays}, {"maxStreamBackendCache", &tsMaxStreamBackendCache}, @@ -2256,15 +2296,31 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"ttlBatchDropNum", &tsTtlBatchDropNum}, {"ttlFlushThreshold", &tsTtlFlushThreshold}, {"ttlPushInterval", &tsTtlPushIntervalSec}, + {"ttlUnit", &tsTtlUnit}, + {"udf", &tsStartUdfd}, + {"udfdLdLibPath", &tsUdfdLdLibPath}, + {"udfdResFuncs", &tsUdfdResFuncs}, + {"s3Accesskey", &tsS3AccessKey}, + {"s3BucketName", &tsS3BucketName}, + {"s3Endpoint", &tsS3Endpoint}, {"s3MigrateIntervalSec", &tsS3MigrateIntervalSec}, {"s3MigrateEnabled", &tsS3MigrateEnabled}, //{"s3BlockSize", &tsS3BlockSize}, {"s3BlockCacheSize", &tsS3BlockCacheSize}, {"s3PageCacheSize", &tsS3PageCacheSize}, {"s3UploadDelaySec", &tsS3UploadDelaySec}, + {"mndSdbWriteDelta", &tsMndSdbWriteDelta}, + {"mndLogRetention", &tsMndLogRetention}, {"supportVnodes", &tsNumOfSupportVnodes}, {"experimental", &tsExperimental}, {"maxTsmaNum", &tsMaxTsmaNum}, + {"maxShellConns", &tsMaxShellConns}, + {"numOfRpcSessions", &tsNumOfRpcSessions}, + {"numOfRpcThreads", &tsNumOfRpcThreads}, + {"rpcQueueMemoryAllowed", &tsQueueMemoryAllowed}, + {"shellActivityTimer", &tsShellActivityTimer}, + {"timeToGetAvailableConn", &tsTimeToGetAvailableConn}, + {"tsReadTimeout", &tsReadTimeout}, {"safetyCheckLevel", &tsSafetyCheckLevel}}; if ((code = taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true)) != TSDB_CODE_SUCCESS) { diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index c106278009..105f597d22 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -453,7 +453,7 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { SConfig *pCfg = taosGetCfg(); SConfigItem *pItem = NULL; - code = cfgGetAndSetItem(pCfg, pItem, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_CMD, true); + code = cfgGetAndSetItem(pCfg, &pItem, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_CMD, true); if (code != 0) { if (strncasecmp(cfgReq.config, "resetlog", strlen("resetlog")) == 0) { code = 0; diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c index e5e6a95c8d..a8880ce94c 100644 --- a/source/dnode/mnode/impl/src/mndConfig.c +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -25,6 +25,11 @@ #define CFG_VER_NUMBER 1 #define CFG_RESERVE_SIZE 63 +enum CfgAlterType { + CFG_ALTER_DNODE, + CFG_ALTER_ALL_DNODES, +}; + static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pInMCfgReq, int32_t optLen, int32_t *pOutValue); static int32_t cfgUpdateItem(SConfigItem *pItem, SConfigObj *obj); static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue); @@ -249,16 +254,15 @@ int32_t mndInitWriteCfg(SMnode *pMnode) { int code = -1; size_t sz = 0; - SConfigObj obj = {0}; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "init-write-config"); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "init-write-config"); if (pTrans == NULL) { mError("failed to init write cfg in create trans, since %s", terrstr()); goto _OVER; } // encode mnd config version - obj = (SConfigObj){.name = "tsmmConfigVersion", .dtype = CFG_DTYPE_INT32, .i32 = tsmmConfigVersion}; - if ((code = mndSetCreateConfigCommitLogs(pTrans, &obj)) != 0) { + SConfigObj *obj = mndInitConfigVersion(); + if ((code = mndSetCreateConfigCommitLogs(pTrans, obj)) != 0) { mError("failed to init mnd config version, since %s", terrstr()); } sz = taosArrayGetSize(taosGetGlobalCfg(tsCfg)); @@ -479,9 +483,11 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { updateIpWhiteList = 1; } - TAOS_CHECK_GOTO(cfgCheckRangeForDynUpdate(taosGetCfg(), dcfgReq.config, dcfgReq.value, true), &lino, _err_out); + bool isUpdateAll = (cfgReq.dnodeId == 0 || cfgReq.dnodeId == -1) ? true : false; + TAOS_CHECK_GOTO(cfgCheckRangeForDynUpdate(taosGetCfg(), dcfgReq.config, dcfgReq.value, true, isUpdateAll), &lino, + _err_out); } - // update config in sdb + // Update config in sdb. TAOS_CHECK_GOTO(mndConfigUpdateTrans(pMnode, cfgReq.config, cfgReq.value), &lino, _err_out); { // audit char obj[50] = {0}; @@ -538,11 +544,13 @@ _err: static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue) { int32_t code = -1; int32_t lino = -1; - SConfigObj *pVersion = mndInitConfigVersion(); + SConfigObj *pVersion = sdbAcquire(pMnode->pSdb, SDB_CFG, "tsmmConfigVersion"); if (pVersion == NULL) { + mWarn("failed to acquire tsmmConfigVersion while update config, since %s", terrstr()); code = terrno; goto _OVER; } + pVersion->i32 = ++tsmmConfigVersion; SConfigObj *pObj = sdbAcquire(pMnode->pSdb, SDB_CFG, name); if (pObj == NULL) { mWarn("failed to acquire mnd config:%s while update config, since %s", name, terrstr()); @@ -555,7 +563,7 @@ static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pVal if (terrno != 0) code = terrno; goto _OVER; } - mInfo("trans:%d, used to update config:%s to value:%s", pTrans->id, pObj->name, pObj->str); + mInfo("trans:%d, used to update config:%s to value:%s", pTrans->id, name, pValue); TAOS_CHECK_GOTO(mndSetCreateConfigCommitLogs(pTrans, pVersion), &lino, _OVER); TAOS_CHECK_GOTO(mndSetCreateConfigCommitLogs(pTrans, pObj), &lino, _OVER); if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 6272ac7049..678d0844a8 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "command.h" #include "catalog.h" +#include "command.h" #include "commandInt.h" #include "scheduler.h" #include "systable.h" @@ -51,7 +51,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->numOfCols = htonl(numOfCols); int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, dataEncodeBufSize, numOfCols); - if(len < 0) { + if (len < 0) { taosMemoryFree(*pRsp); return terrno; } @@ -898,7 +898,7 @@ static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { goto _return; } - if (cfgCheckRangeForDynUpdate(tsCfg, pStmt->config, pStmt->value, false)) { + if (cfgCheckRangeForDynUpdate(tsCfg, pStmt->config, pStmt->value, false, false)) { return terrno; } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index f05c90f6a3..72dcd8f284 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -40,7 +40,7 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile); int32_t cfgLoadFromEnvVar(SConfig *pConfig); int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd); int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url); -int32_t cfgSetItemVal(SConfigItem *item, ECfgDataType dtype); +int32_t cfgSetItemVal(SConfigItem *pItem, const char *name, const char *value, ECfgSrcType stype); extern char **environ; @@ -354,7 +354,7 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); } - TAOS_CHECK_RETURN(cfgSetItemVal(pItem, pItem->dtype)); + TAOS_CHECK_RETURN(cfgSetItemVal(pItem, name, value, stype)); if (lock) { (void)taosThreadMutexUnlock(&pCfg->lock); @@ -363,7 +363,7 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy TAOS_RETURN(code); } -int32_t cfgGetAndSetItem(SConfig *pCfg, SConfigItem *pItem, const char *name, const char *value, ECfgSrcType stype, +int32_t cfgGetAndSetItem(SConfig *pCfg, SConfigItem **pItem, const char *name, const char *value, ECfgSrcType stype, bool lock) { // GRANT_CFG_SET; int32_t code = TSDB_CODE_SUCCESS; @@ -372,13 +372,13 @@ int32_t cfgGetAndSetItem(SConfig *pCfg, SConfigItem *pItem, const char *name, co (void)taosThreadMutexLock(&pCfg->lock); } - pItem = cfgGetItem(pCfg, name); + *pItem = cfgGetItem(pCfg, name); if (pItem == NULL) { (void)taosThreadMutexUnlock(&pCfg->lock); TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); } - TAOS_CHECK_RETURN(cfgSetItemVal(pItem, pItem->dtype)); + TAOS_CHECK_RETURN(cfgSetItemVal(*pItem, name, value, stype)); if (lock) { (void)taosThreadMutexUnlock(&pCfg->lock); @@ -387,34 +387,54 @@ int32_t cfgGetAndSetItem(SConfig *pCfg, SConfigItem *pItem, const char *name, co TAOS_RETURN(code); } -int32_t cfgSetItemVal(SConfigItem *item, ECfgDataType dtype) { - if (item == NULL) { +int32_t cfgSetItemVal(SConfigItem *pItem, const char *name, const char *value, ECfgSrcType stype) { + int32_t code = TSDB_CODE_SUCCESS; + + if (pItem == NULL) { TAOS_RETURN(TSDB_CODE_INVALID_CFG); } - item->dtype = dtype; - switch (dtype) { - case CFG_DTYPE_BOOL: - item->bval = false; + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: { + code = cfgSetBool(pItem, value, stype); break; - case CFG_DTYPE_INT32: - item->i32 = 0; + } + case CFG_DTYPE_INT32: { + code = cfgSetInt32(pItem, value, stype); break; - case CFG_DTYPE_INT64: - item->i64 = 0; + } + case CFG_DTYPE_INT64: { + code = cfgSetInt64(pItem, value, stype); break; + } case CFG_DTYPE_FLOAT: - case CFG_DTYPE_DOUBLE: - item->fval = 0; + case CFG_DTYPE_DOUBLE: { + code = cfgSetFloat(pItem, value, stype); break; - case CFG_DTYPE_STRING: - case CFG_DTYPE_DIR: - case CFG_DTYPE_LOCALE: - case CFG_DTYPE_CHARSET: - case CFG_DTYPE_TIMEZONE: - item->str = NULL; + } + case CFG_DTYPE_STRING: { + code = cfgSetString(pItem, value, stype); break; + } + case CFG_DTYPE_DIR: { + code = cfgSetDir(pItem, value, stype); + break; + } + case CFG_DTYPE_TIMEZONE: { + code = cfgSetTimezone(pItem, value, stype); + break; + } + case CFG_DTYPE_CHARSET: { + code = doSetConf(pItem, value, stype); + break; + } + case CFG_DTYPE_LOCALE: { + code = doSetConf(pItem, value, stype); + break; + } + case CFG_DTYPE_NONE: default: - TAOS_RETURN(TSDB_CODE_INVALID_CFG); + code = TSDB_CODE_INVALID_CFG; + break; } TAOS_RETURN(TSDB_CODE_SUCCESS); @@ -450,7 +470,7 @@ void cfgLock(SConfig *pCfg) { void cfgUnLock(SConfig *pCfg) { (void)taosThreadMutexUnlock(&pCfg->lock); } -int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer) { +int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer, bool isUpdateAll) { ECfgDynType dynType = isServer ? CFG_DYN_SERVER : CFG_DYN_CLIENT; cfgLock(pCfg); @@ -461,6 +481,11 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p cfgUnLock(pCfg); TAOS_RETURN(TSDB_CODE_INVALID_CFG); } + if (!isUpdateAll && (pItem->category & CFG_CATEGORY_GLOBAL) == 0) { + uError("failed to config:%s, not support update global config on only one dnode", name); + cfgUnLock(pCfg); + TAOS_RETURN(TSDB_CODE_INVALID_CFG); + } switch (pItem->dtype) { case CFG_DTYPE_STRING: {