From 404b66e860689d0d9df679abe55fac135eac225e Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 27 Nov 2024 16:27:14 +0800 Subject: [PATCH] feat: support auto compact --- include/common/tmsg.h | 16 ++++++---- source/common/src/tmsg.c | 12 ++++---- source/libs/parser/inc/sql.y | 42 ++++++++++++++++++++------ source/libs/parser/src/parAstCreater.c | 9 +++--- source/libs/parser/src/parTranslater.c | 22 ++++++++------ 5 files changed, 66 insertions(+), 35 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index cdc43ad64a..ec1bcdc269 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1343,10 +1343,10 @@ typedef struct { int8_t encryptAlgorithm; char dnodeListStr[TSDB_DNODE_LIST_LEN]; // 1. add auto-compact parameters - int32_t compactInterval; - int64_t compactStartTime; - int64_t compactEndTime; - int8_t compactTimeOffset; + int32_t compactInterval; // minutes + int32_t compactStartTime; // minutes + int32_t compactEndTime; // minutes + int8_t compactTimeOffset; // hour } SCreateDbReq; int32_t tSerializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq); @@ -1380,8 +1380,8 @@ typedef struct { int8_t withArbitrator; // 1. add auto-compact parameters int32_t compactInterval; - int64_t compactStartTime; - int64_t compactEndTime; + int32_t compactStartTime; + int32_t compactEndTime; int8_t compactTimeOffset; } SAlterDbReq; @@ -1515,6 +1515,10 @@ typedef struct { int32_t s3ChunkSize; int32_t s3KeepLocal; int8_t s3Compact; + int8_t compactTimeOffset; + int32_t compactInterval; + int32_t compactStartTime; + int32_t compactEndTime; int32_t tsdbPageSize; int32_t walRetentionPeriod; int32_t walRollPeriod; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index eff7e4c2ea..730db800c5 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4036,8 +4036,8 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) if (!tDecodeIsEnd(&decoder)) { TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactInterval)); - TAOS_CHECK_EXIT(tDecodeI64v(&decoder, &pReq->compactStartTime)); - TAOS_CHECK_EXIT(tDecodeI64v(&decoder, &pReq->compactEndTime)); + TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactStartTime)); + TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactEndTime)); TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->compactTimeOffset)); } else { pReq->compactInterval = 0; @@ -4098,8 +4098,8 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->withArbitrator)); // auto compact config TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pReq->compactInterval)); - TAOS_CHECK_EXIT(tEncodeI64v(&encoder, pReq->compactStartTime)); - TAOS_CHECK_EXIT(tEncodeI64v(&encoder, pReq->compactEndTime)); + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pReq->compactStartTime)); + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pReq->compactEndTime)); TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->compactTimeOffset)); tEndEncode(&encoder); @@ -4172,8 +4172,8 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { // auto compact config if (!tDecodeIsEnd(&decoder)) { TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactInterval)); - TAOS_CHECK_EXIT(tDecodeI64v(&decoder, &pReq->compactStartTime)); - TAOS_CHECK_EXIT(tDecodeI64v(&decoder, &pReq->compactEndTime)); + TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactStartTime)); + TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactEndTime)); TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->compactTimeOffset)); } else { pReq->compactInterval = 0; diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 869e925799..cb0e58be29 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -290,8 +290,8 @@ db_options(A) ::= db_options(B) ENCRYPT_ALGORITHM NK_STRING(C). db_options(A) ::= db_options(B) DNODES NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DNODES, &C); } db_options(A) ::= db_options(B) COMPACT_INTERVAL NK_INTEGER (C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_INTERVAL, &C); } db_options(A) ::= db_options(B) COMPACT_INTERVAL NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_INTERVAL, &C); } -db_options(A) ::= db_options(B) COMPACT_TIME_RANGE integer_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_RANGE, &C); } -db_options(A) ::= db_options(B) COMPACT_TIME_RANGE variable_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_RANGE, &C); } +db_options(A) ::= db_options(B) COMPACT_TIME_RANGE integer_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_RANGE, C); } +db_options(A) ::= db_options(B) COMPACT_TIME_RANGE variable_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_RANGE, C); } db_options(A) ::= db_options(B) COMPACT_TIME_OFFSET NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_OFFSET, &C); } db_options(A) ::= db_options(B) COMPACT_TIME_OFFSET NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_OFFSET, &C); } @@ -329,8 +329,10 @@ alter_db_option(A) ::= S3_KEEPLOCAL NK_VARIABLE(B). alter_db_option(A) ::= S3_COMPACT NK_INTEGER(B). { A.type = DB_OPTION_S3_COMPACT, A.val = B; } alter_db_option(A) ::= KEEP_TIME_OFFSET NK_INTEGER(B). { A.type = DB_OPTION_KEEP_TIME_OFFSET; A.val = B; } alter_db_option(A) ::= ENCRYPT_ALGORITHM NK_STRING(B). { A.type = DB_OPTION_ENCRYPT_ALGORITHM; A.val = B; } +alter_db_option(A) ::= COMPACT_INTERVAL NK_INTEGER(B). { A.type = DB_OPTION_COMPACT_INTERVAL; A.val = B; } alter_db_option(A) ::= COMPACT_INTERVAL NK_VARIABLE(B). { A.type = DB_OPTION_COMPACT_INTERVAL; A.val = B; } -alter_db_option(A) ::= COMPACT_TIME_RANGE variable_list(B). { A.type = DB_OPTION_COMPACT_TIME_RANGE; A.pList = B; } +alter_db_option(A) ::= COMPACT_TIME_RANGE signed_integer_list(B). { A.type = DB_OPTION_COMPACT_TIME_RANGE; A.pList = B; } +alter_db_option(A) ::= COMPACT_TIME_RANGE signed_variable_list(B). { A.type = DB_OPTION_COMPACT_TIME_RANGE; A.pList = B; } alter_db_option(A) ::= COMPACT_TIME_OFFSET NK_INTEGER(B). { A.type = DB_OPTION_COMPACT_TIME_OFFSET; A.val = B; } %type integer_list { SNodeList* } @@ -338,11 +340,21 @@ alter_db_option(A) ::= COMPACT_TIME_OFFSET NK_INTEGER(B). integer_list(A) ::= NK_INTEGER(B). { A = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); } integer_list(A) ::= integer_list(B) NK_COMMA NK_INTEGER(C). { A = addNodeToList(pCxt, B, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C)); } +%type signed_integer_list { SNodeList* } +%destructor signed_integer_list { nodesDestroyList($$); } +signed_integer_list(A) ::= signed_integer(B). { A = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); } +signed_integer_list(A) ::= signed_integer_list(B) NK_COMMA signed_integer(C). { A = addNodeToList(pCxt, B, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C)); } + %type variable_list { SNodeList* } %destructor variable_list { nodesDestroyList($$); } variable_list(A) ::= NK_VARIABLE(B). { A = createNodeList(pCxt, createDurationValueNode(pCxt, &B)); } variable_list(A) ::= variable_list(B) NK_COMMA NK_VARIABLE(C). { A = addNodeToList(pCxt, B, createDurationValueNode(pCxt, &C)); } +%type signed_variable_list { SNodeList* } +%destructor signed_variable_list { nodesDestroyList($$); } +signed_variable_list(A) ::= NK_VARIABLE(B). { A = createNodeList(pCxt, createDurationValueNode(pCxt, &B)); } +signed_variable_list(A) ::= variable_list(B) NK_COMMA NK_VARIABLE(C). { A = addNodeToList(pCxt, B, createDurationValueNode(pCxt, &C)); } + %type retention_list { SNodeList* } %destructor retention_list { nodesDestroyList($$); } retention_list(A) ::= retention(B). { A = createNodeList(pCxt, B); } @@ -1044,21 +1056,33 @@ literal(A) ::= NK_QUESTION(B). duration_literal(A) ::= NK_VARIABLE(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); } -signed(A) ::= NK_INTEGER(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &B); } -signed(A) ::= NK_PLUS NK_INTEGER(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &B); } -signed(A) ::= NK_MINUS(B) NK_INTEGER(C). { +signed_variable(A) ::= NK_VARIABLE(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); } +signed_variable(A) ::= NK_PLUS NK_VARIABLE(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); } +signed_variable(A) ::= NK_MINUS(B) NK_VARIABLE(C). { + SToken t = B; + t.n = (C.z + C.n) - B.z; + A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &t)); + } + +signed_integer(A) ::= NK_INTEGER(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &B); } +signed_integer(A) ::= NK_PLUS NK_INTEGER(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &B); } +signed_integer(A) ::= NK_MINUS(B) NK_INTEGER(C). { SToken t = B; t.n = (C.z + C.n) - B.z; A = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); } -signed(A) ::= NK_FLOAT(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); } -signed(A) ::= NK_PLUS NK_FLOAT(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); } -signed(A) ::= NK_MINUS(B) NK_FLOAT(C). { + +signed_float(A) ::= NK_FLOAT(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); } +signed_float(A) ::= NK_PLUS NK_FLOAT(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); } +signed_float(A) ::= NK_MINUS(B) NK_FLOAT(C). { SToken t = B; t.n = (C.z + C.n) - B.z; A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); } +signed(A) ::= signed_integer(B). { A = B; } +signed(A) ::= signed_float(B). { A = B; } + signed_literal(A) ::= signed(B). { A = B; } signed_literal(A) ::= NK_STRING(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B); } signed_literal(A) ::= NK_BOOL(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 720a26fa45..fe5bee265b 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1848,10 +1848,10 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) { pOptions->withArbitrator = -1; pOptions->encryptAlgorithm = -1; pOptions->dnodeListStr[0] = 0; - pOptions->compactInterval = TSDB_DEFAULT_COMPACT_INTERVAL; - pOptions->compactStartTime = TSDB_DEFAULT_COMPACT_START_TIME; - pOptions->compactEndTime = TSDB_DEFAULT_COMPACT_END_TIME; - pOptions->compactTimeOffset = TSDB_DEFAULT_COMPACT_TIME_OFFSET; + pOptions->compactInterval = -1; + pOptions->compactStartTime = -1; + pOptions->compactEndTime = -1; + pOptions->compactTimeOffset = -1; return (SNode*)pOptions; _err: return NULL; @@ -2033,6 +2033,7 @@ SNode* setAlterDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOp switch (pAlterOption->type) { case DB_OPTION_KEEP: case DB_OPTION_RETENTIONS: + case DB_OPTION_COMPACT_TIME_RANGE: return setDatabaseOptionImpl(pCxt, pOptions, pAlterOption->type, pAlterOption->pList, true); default: break; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 9391c08a81..cd65f0afdb 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7989,26 +7989,28 @@ static int32_t checkDbCompactTimeRangeOption(STranslateContext* pCxt, SDatabaseO pEnd->unit, TIME_UNIT_MINUTE, TIME_UNIT_HOUR, TIME_UNIT_DAY); } } else { - pStart->datum.i *= 1440; + pEnd->datum.i *= 1440; } pOptions->compactStartTime = getBigintFromValueNode(pStart); pOptions->compactEndTime = getBigintFromValueNode(pEnd); if (pOptions->compactStartTime >= pOptions->compactEndTime) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, - "Invalid option compact_time_range, start time should be less than end time"); + return generateSyntaxErrMsgExt( + &pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option compact_time_range: %dm,%dm, start time should be less than end time", + pOptions->compactStartTime, pOptions->compactEndTime); } if (pOptions->compactStartTime < -pOptions->keep[2] || pOptions->compactStartTime > -pOptions->daysPerFile) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, - "Invalid option compact_time_range, start_time should be in range: [%" PRIi64 - "m, %" PRId64 "m]", - -pOptions->keep[2], -pOptions->daysPerFile); + "Invalid option compact_time_range: %dm, start_time should be in range: [%" PRIi64 + "m, %dm]", + pOptions->compactStartTime, -pOptions->keep[2], -pOptions->daysPerFile); } if (pOptions->compactEndTime < -pOptions->keep[2] || pOptions->compactEndTime > -pOptions->daysPerFile) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, - "Invalid option compact_time_range, end time should be in range: [%" PRIi64 - "m, %" PRId64 "m]", - -pOptions->keep[2], -pOptions->daysPerFile); + "Invalid option compact_time_range: %dm, end time should be in range: [%" PRIi64 + "m, %dm]", + pOptions->compactEndTime, -pOptions->keep[2], -pOptions->daysPerFile); } return TSDB_CODE_SUCCESS; @@ -8019,7 +8021,7 @@ static int32_t checkDbCompactTimeOffsetOption(STranslateContext* pCxt, SDatabase pOptions->compactTimeOffset > TSDB_MAX_COMPACT_TIME_OFFSET) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, "Invalid option compact_time_offset: %d" - " valid range: [%d, %d]", + ", valid range: [%d, %d]", pOptions->compactTimeOffset, TSDB_MIN_COMPACT_TIME_OFFSET, TSDB_MAX_COMPACT_TIME_OFFSET); }