From 77418f154a92c119e21f55b604d0506783aed0b6 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 4 Dec 2024 10:49:33 +0800 Subject: [PATCH] feat: db options for auto compact --- source/common/src/msg/tmsg.c | 63 +++++++++++++++++--------- source/dnode/mnode/impl/inc/mndDef.h | 4 +- source/dnode/mnode/impl/src/mndDb.c | 14 ++++-- source/libs/command/src/command.c | 6 ++- source/libs/parser/src/parTranslater.c | 4 +- 5 files changed, 58 insertions(+), 33 deletions(-) diff --git a/source/common/src/msg/tmsg.c b/source/common/src/msg/tmsg.c index f26ae89743..26ac61c42a 100644 --- a/source/common/src/msg/tmsg.c +++ b/source/common/src/msg/tmsg.c @@ -3939,8 +3939,8 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) { // auto-compact parameters TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pReq->compactInterval)); - TAOS_CHECK_EXIT(tEncodeI64v(&encoder, pReq->compactStartTime)); - TAOS_CHECK_EXIT(tEncodeI64v(&encoder, pReq->compactEndTime)); + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pReq->compactStartTime)); + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pReq->compactEndTime)); TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->compactTimeOffset)); tEndEncode(&encoder); @@ -4010,24 +4010,26 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->tsdbPageSize)); - pReq->keepTimeOffset = TSDB_DEFAULT_KEEP_TIME_OFFSET; if (!tDecodeIsEnd(&decoder)) { TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->keepTimeOffset)); + } else { + pReq->keepTimeOffset = TSDB_DEFAULT_KEEP_TIME_OFFSET; } DECODESQL(); - pReq->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; - pReq->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO; - pReq->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; - pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; - pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT; if (!tDecodeIsEnd(&decoder)) { TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->withArbitrator)); TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->encryptAlgorithm)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->s3ChunkSize)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->s3KeepLocal)); TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->s3Compact)); + } else { + pReq->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; + pReq->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO; + pReq->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; + pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; + pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT; } if (!tDecodeIsEnd(&decoder)) { @@ -4040,10 +4042,10 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactEndTime)); TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->compactTimeOffset)); } else { - pReq->compactInterval = 0; - pReq->compactStartTime = 0; - pReq->compactEndTime = 0; - pReq->compactTimeOffset = 0; + pReq->compactInterval = TSDB_DEFAULT_COMPACT_INTERVAL; + pReq->compactStartTime = TSDB_DEFAULT_COMPACT_START_TIME; + pReq->compactEndTime = TSDB_DEFAULT_COMPACT_END_TIME; + pReq->compactTimeOffset = TSDB_DEFAULT_COMPACT_TIME_OFFSET; } tEndDecode(&decoder); @@ -4176,10 +4178,10 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactEndTime)); TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->compactTimeOffset)); } else { - pReq->compactInterval = 0; - pReq->compactStartTime = 0; - pReq->compactEndTime = 0; - pReq->compactTimeOffset = 0; + pReq->compactInterval = TSDB_DEFAULT_COMPACT_INTERVAL; + pReq->compactStartTime = TSDB_DEFAULT_COMPACT_START_TIME; + pReq->compactEndTime = TSDB_DEFAULT_COMPACT_END_TIME; + pReq->compactTimeOffset = TSDB_DEFAULT_COMPACT_TIME_OFFSET; } tEndDecode(&decoder); @@ -5314,6 +5316,10 @@ int32_t tSerializeSDbCfgRspImpl(SEncoder *encoder, const SDbCfgRsp *pRsp) { TAOS_CHECK_RETURN(tEncodeI32(encoder, pRsp->s3KeepLocal)); TAOS_CHECK_RETURN(tEncodeI8(encoder, pRsp->s3Compact)); TAOS_CHECK_RETURN(tEncodeI8(encoder, pRsp->hashMethod)); + TAOS_CHECK_RETURN(tEncodeI32v(encoder, pRsp->compactInterval)); + TAOS_CHECK_RETURN(tEncodeI32v(encoder, pRsp->compactStartTime)); + TAOS_CHECK_RETURN(tEncodeI32v(encoder, pRsp->compactEndTime)); + TAOS_CHECK_RETURN(tEncodeI8(encoder, pRsp->compactTimeOffset)); return 0; } @@ -5389,27 +5395,40 @@ int32_t tDeserializeSDbCfgRspImpl(SDecoder *decoder, SDbCfgRsp *pRsp) { } TAOS_CHECK_RETURN(tDecodeI8(decoder, &pRsp->schemaless)); TAOS_CHECK_RETURN(tDecodeI16(decoder, &pRsp->sstTrigger)); - pRsp->keepTimeOffset = TSDB_DEFAULT_KEEP_TIME_OFFSET; if (!tDecodeIsEnd(decoder)) { TAOS_CHECK_RETURN(tDecodeI32(decoder, &pRsp->keepTimeOffset)); + } else { + pRsp->keepTimeOffset = TSDB_DEFAULT_KEEP_TIME_OFFSET; } - pRsp->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; - pRsp->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO; - pRsp->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; - pRsp->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; - pRsp->s3Compact = TSDB_DEFAULT_S3_COMPACT; if (!tDecodeIsEnd(decoder)) { TAOS_CHECK_RETURN(tDecodeI8(decoder, &pRsp->withArbitrator)); TAOS_CHECK_RETURN(tDecodeI8(decoder, &pRsp->encryptAlgorithm)); TAOS_CHECK_RETURN(tDecodeI32(decoder, &pRsp->s3ChunkSize)); TAOS_CHECK_RETURN(tDecodeI32(decoder, &pRsp->s3KeepLocal)); TAOS_CHECK_RETURN(tDecodeI8(decoder, &pRsp->s3Compact)); + } else { + pRsp->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; + pRsp->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO; + pRsp->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; + pRsp->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; + pRsp->s3Compact = TSDB_DEFAULT_S3_COMPACT; } if (!tDecodeIsEnd(decoder)) { TAOS_CHECK_RETURN(tDecodeI8(decoder, &pRsp->hashMethod)); } else { pRsp->hashMethod = 1; // default value } + if (!tDecodeIsEnd(decoder)) { + TAOS_CHECK_RETURN(tDecodeI32v(decoder, &pRsp->compactInterval)); + TAOS_CHECK_RETURN(tDecodeI32v(decoder, &pRsp->compactStartTime)); + TAOS_CHECK_RETURN(tDecodeI32v(decoder, &pRsp->compactEndTime)); + TAOS_CHECK_RETURN(tDecodeI8(decoder, &pRsp->compactTimeOffset)); + } else { + pRsp->compactInterval = TSDB_DEFAULT_COMPACT_INTERVAL; + pRsp->compactStartTime = TSDB_DEFAULT_COMPACT_START_TIME; + pRsp->compactEndTime = TSDB_DEFAULT_COMPACT_END_TIME; + pRsp->compactTimeOffset = TSDB_DEFAULT_COMPACT_TIME_OFFSET; + } return 0; } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 66b296dde4..3a47b87b8b 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -425,8 +425,8 @@ typedef struct { int8_t encryptAlgorithm; int8_t compactTimeOffset; int32_t compactInterval; - int64_t compactStartTime; - int64_t compactEndTime; + int32_t compactStartTime; + int32_t compactEndTime; } SDbCfg; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index deddb22907..1c6c54de95 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -37,7 +37,7 @@ #include "tjson.h" #define DB_VER_NUMBER 1 -#define DB_RESERVE_SIZE 6 +#define DB_RESERVE_SIZE 14 static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw); static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb); @@ -152,8 +152,8 @@ SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT8(pRaw, dataPos, pDb->cfg.encryptAlgorithm, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->tsmaVersion, _OVER); SDB_SET_INT8(pRaw, dataPos, pDb->cfg.compactTimeOffset, _OVER) - SDB_SET_INT64(pRaw, dataPos, pDb->cfg.compactStartTime, _OVER) - SDB_SET_INT64(pRaw, dataPos, pDb->cfg.compactEndTime, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.compactStartTime, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.compactEndTime, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.compactInterval, _OVER) SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) @@ -255,8 +255,8 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.encryptAlgorithm, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->tsmaVersion, _OVER); SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.compactTimeOffset, _OVER) - SDB_GET_INT64(pRaw, dataPos, &pDb->cfg.compactStartTime, _OVER) - SDB_GET_INT64(pRaw, dataPos, &pDb->cfg.compactEndTime, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.compactStartTime, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.compactEndTime, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.compactInterval, _OVER) SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) @@ -1413,6 +1413,10 @@ static void mndDumpDbCfgInfo(SDbCfgRsp *cfgRsp, SDbObj *pDb) { cfgRsp->s3Compact = pDb->cfg.s3Compact; cfgRsp->withArbitrator = pDb->cfg.withArbitrator; cfgRsp->encryptAlgorithm = pDb->cfg.encryptAlgorithm; + cfgRsp->compactInterval = pDb->cfg.compactInterval; + cfgRsp->compactStartTime = pDb->cfg.compactStartTime; + cfgRsp->compactEndTime = pDb->cfg.compactEndTime; + cfgRsp->compactTimeOffset = pDb->cfg.compactTimeOffset; } static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) { diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 353bc1fa18..ae1bcb461e 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -428,13 +428,15 @@ static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, "PRECISION '%s' REPLICA %d " "WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d TABLE_PREFIX %d TABLE_SUFFIX %d TSDB_PAGESIZE %d " "WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64 - " KEEP_TIME_OFFSET %d ENCRYPT_ALGORITHM '%s' S3_CHUNKPAGES %d S3_KEEPLOCAL %dm S3_COMPACT %d", + " KEEP_TIME_OFFSET %d ENCRYPT_ALGORITHM '%s' S3_CHUNKPAGES %d S3_KEEPLOCAL %dm S3_COMPACT %d " + "COMPACT_INTERVAL %d COMPACT_TIME_RANGE %d,%d COMPACT_TIME_OFFSET %"PRIi8, dbName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, durationStr, pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->sstTrigger, keep0Str, keep1Str, keep2Str, pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups, 1 == pCfg->numOfStables, hashPrefix, pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod, pCfg->walRetentionSize, pCfg->keepTimeOffset, encryptAlgorithmStr(pCfg->encryptAlgorithm), - pCfg->s3ChunkSize, pCfg->s3KeepLocal, pCfg->s3Compact); + pCfg->s3ChunkSize, pCfg->s3KeepLocal, pCfg->s3Compact, pCfg->compactInterval, pCfg->compactStartTime, + pCfg->compactEndTime, pCfg->compactTimeOffset); if (pRetentions) { len += tsnprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_DB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index bf3345addc..6ebc720aa3 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2002,7 +2002,7 @@ static int32_t parseBoolFromValueNode(STranslateContext* pCxt, SValueNode* pVal) static EDealRes translateDurationValue(STranslateContext* pCxt, SValueNode* pVal) { if (parseNatualDuration(pVal->literal, strlen(pVal->literal), &pVal->datum.i, &pVal->unit, - pVal->node.resType.precision, false) != TSDB_CODE_SUCCESS) { + pVal->node.resType.precision, true) != TSDB_CODE_SUCCESS) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } *(int64_t*)&pVal->typeData = pVal->datum.i; @@ -8099,7 +8099,7 @@ static int32_t checkDbCompactIntervalOption(STranslateContext* pCxt, SDatabaseOp } interval = getBigintFromValueNode(pOptions->pCompactIntervalNode); if (interval != 0) { - code = checkDbRangeOption(pCxt, "compact_interval", pOptions->compactInterval, TSDB_MIN_COMPACT_INTERVAL, + code = checkDbRangeOption(pCxt, "compact_interval", interval, TSDB_MIN_COMPACT_INTERVAL, pOptions->keep[2]); } } else if (pOptions->compactInterval != 0) {