xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: support auto compact

This commit is contained in:
kailixu 2024-11-27 16:27:14 +08:00
parent 67924ee55b
commit 404b66e860
5 changed files with 66 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
include/common/tmsg.h
View File

@ -1343,10 +1343,10 @@ typedef struct {
int8_t encryptAlgorithm;
char dnodeListStr[TSDB_DNODE_LIST_LEN];
// 1. add auto-compact parameters
int32_t compactInterval;
int64_t compactStartTime;
int64_t compactEndTime;
int8_t compactTimeOffset;
int32_t compactInterval; // minutes
int32_t compactStartTime; // minutes
int32_t compactEndTime; // minutes
int8_t compactTimeOffset; // hour
} SCreateDbReq;
int32_t tSerializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq);
@ -1380,8 +1380,8 @@ typedef struct {
int8_t withArbitrator;
// 1. add auto-compact parameters
int32_t compactInterval;
int64_t compactStartTime;
int64_t compactEndTime;
int32_t compactStartTime;
int32_t compactEndTime;
int8_t compactTimeOffset;
} SAlterDbReq;
@ -1515,6 +1515,10 @@ typedef struct {
int32_t s3ChunkSize;
int32_t s3KeepLocal;
int8_t s3Compact;
int8_t compactTimeOffset;
int32_t compactInterval;
int32_t compactStartTime;
int32_t compactEndTime;
int32_t tsdbPageSize;
int32_t walRetentionPeriod;
int32_t walRollPeriod;

12
source/common/src/tmsg.c
View File

@ -4036,8 +4036,8 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq)
if (!tDecodeIsEnd(&decoder)) {
TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactInterval));
TAOS_CHECK_EXIT(tDecodeI64v(&decoder, &pReq->compactStartTime));
TAOS_CHECK_EXIT(tDecodeI64v(&decoder, &pReq->compactEndTime));
TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactStartTime));
TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactEndTime));
TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->compactTimeOffset));
} else {
pReq->compactInterval = 0;
@ -4098,8 +4098,8 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->withArbitrator));
// auto compact config
TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pReq->compactInterval));
TAOS_CHECK_EXIT(tEncodeI64v(&encoder, pReq->compactStartTime));
TAOS_CHECK_EXIT(tEncodeI64v(&encoder, pReq->compactEndTime));
TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pReq->compactStartTime));
TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pReq->compactEndTime));
TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->compactTimeOffset));
tEndEncode(&encoder);
@ -4172,8 +4172,8 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
// auto compact config
if (!tDecodeIsEnd(&decoder)) {
TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactInterval));
TAOS_CHECK_EXIT(tDecodeI64v(&decoder, &pReq->compactStartTime));
TAOS_CHECK_EXIT(tDecodeI64v(&decoder, &pReq->compactEndTime));
TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactStartTime));
TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactEndTime));
TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->compactTimeOffset));
} else {
pReq->compactInterval = 0;

42
source/libs/parser/inc/sql.y
View File

@ -290,8 +290,8 @@ db_options(A) ::= db_options(B) ENCRYPT_ALGORITHM NK_STRING(C).
db_options(A) ::= db_options(B) DNODES NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DNODES, &C); }
db_options(A) ::= db_options(B) COMPACT_INTERVAL NK_INTEGER (C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_INTERVAL, &C); }
db_options(A) ::= db_options(B) COMPACT_INTERVAL NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_INTERVAL, &C); }
db_options(A) ::= db_options(B) COMPACT_TIME_RANGE integer_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_RANGE, &C); }
db_options(A) ::= db_options(B) COMPACT_TIME_RANGE variable_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_RANGE, &C); }
db_options(A) ::= db_options(B) COMPACT_TIME_RANGE integer_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_RANGE, C); }
db_options(A) ::= db_options(B) COMPACT_TIME_RANGE variable_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_RANGE, C); }
db_options(A) ::= db_options(B) COMPACT_TIME_OFFSET NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_OFFSET, &C); }
db_options(A) ::= db_options(B) COMPACT_TIME_OFFSET NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_OFFSET, &C); }
@ -329,8 +329,10 @@ alter_db_option(A) ::= S3_KEEPLOCAL NK_VARIABLE(B).
alter_db_option(A) ::= S3_COMPACT NK_INTEGER(B). { A.type = DB_OPTION_S3_COMPACT, A.val = B; }
alter_db_option(A) ::= KEEP_TIME_OFFSET NK_INTEGER(B). { A.type = DB_OPTION_KEEP_TIME_OFFSET; A.val = B; }
alter_db_option(A) ::= ENCRYPT_ALGORITHM NK_STRING(B). { A.type = DB_OPTION_ENCRYPT_ALGORITHM; A.val = B; }
alter_db_option(A) ::= COMPACT_INTERVAL NK_INTEGER(B). { A.type = DB_OPTION_COMPACT_INTERVAL; A.val = B; }
alter_db_option(A) ::= COMPACT_INTERVAL NK_VARIABLE(B). { A.type = DB_OPTION_COMPACT_INTERVAL; A.val = B; }
alter_db_option(A) ::= COMPACT_TIME_RANGE variable_list(B). { A.type = DB_OPTION_COMPACT_TIME_RANGE; A.pList = B; }
alter_db_option(A) ::= COMPACT_TIME_RANGE signed_integer_list(B). { A.type = DB_OPTION_COMPACT_TIME_RANGE; A.pList = B; }
alter_db_option(A) ::= COMPACT_TIME_RANGE signed_variable_list(B). { A.type = DB_OPTION_COMPACT_TIME_RANGE; A.pList = B; }
alter_db_option(A) ::= COMPACT_TIME_OFFSET NK_INTEGER(B). { A.type = DB_OPTION_COMPACT_TIME_OFFSET; A.val = B; }
%type integer_list { SNodeList* }
@ -338,11 +340,21 @@ alter_db_option(A) ::= COMPACT_TIME_OFFSET NK_INTEGER(B).
integer_list(A) ::= NK_INTEGER(B). { A = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); }
integer_list(A) ::= integer_list(B) NK_COMMA NK_INTEGER(C). { A = addNodeToList(pCxt, B, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C)); }
%type signed_integer_list { SNodeList* }
%destructor signed_integer_list { nodesDestroyList($$); }
signed_integer_list(A) ::= signed_integer(B). { A = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); }
signed_integer_list(A) ::= signed_integer_list(B) NK_COMMA signed_integer(C). { A = addNodeToList(pCxt, B, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C)); }
%type variable_list { SNodeList* }
%destructor variable_list { nodesDestroyList($$); }
variable_list(A) ::= NK_VARIABLE(B). { A = createNodeList(pCxt, createDurationValueNode(pCxt, &B)); }
variable_list(A) ::= variable_list(B) NK_COMMA NK_VARIABLE(C). { A = addNodeToList(pCxt, B, createDurationValueNode(pCxt, &C)); }
%type signed_variable_list { SNodeList* }
%destructor signed_variable_list { nodesDestroyList($$); }
signed_variable_list(A) ::= NK_VARIABLE(B). { A = createNodeList(pCxt, createDurationValueNode(pCxt, &B)); }
signed_variable_list(A) ::= variable_list(B) NK_COMMA NK_VARIABLE(C). { A = addNodeToList(pCxt, B, createDurationValueNode(pCxt, &C)); }
%type retention_list { SNodeList* }
%destructor retention_list { nodesDestroyList($$); }
retention_list(A) ::= retention(B). { A = createNodeList(pCxt, B); }
@ -1044,21 +1056,33 @@ literal(A) ::= NK_QUESTION(B).
duration_literal(A) ::= NK_VARIABLE(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); }
signed(A) ::= NK_INTEGER(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &B); }
signed(A) ::= NK_PLUS NK_INTEGER(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &B); }
signed(A) ::= NK_MINUS(B) NK_INTEGER(C). {
signed_variable(A) ::= NK_VARIABLE(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); }
signed_variable(A) ::= NK_PLUS NK_VARIABLE(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); }
signed_variable(A) ::= NK_MINUS(B) NK_VARIABLE(C). {
SToken t = B;
t.n = (C.z + C.n) - B.z;
A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &t));
}
signed_integer(A) ::= NK_INTEGER(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &B); }
signed_integer(A) ::= NK_PLUS NK_INTEGER(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &B); }
signed_integer(A) ::= NK_MINUS(B) NK_INTEGER(C). {
SToken t = B;
t.n = (C.z + C.n) - B.z;
A = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t);
}
signed(A) ::= NK_FLOAT(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); }
signed(A) ::= NK_PLUS NK_FLOAT(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); }
signed(A) ::= NK_MINUS(B) NK_FLOAT(C). {
signed_float(A) ::= NK_FLOAT(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); }
signed_float(A) ::= NK_PLUS NK_FLOAT(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); }
signed_float(A) ::= NK_MINUS(B) NK_FLOAT(C). {
SToken t = B;
t.n = (C.z + C.n) - B.z;
A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t);
}
signed(A) ::= signed_integer(B). { A = B; }
signed(A) ::= signed_float(B). { A = B; }
signed_literal(A) ::= signed(B). { A = B; }
signed_literal(A) ::= NK_STRING(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B); }
signed_literal(A) ::= NK_BOOL(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B); }

9
source/libs/parser/src/parAstCreater.c
View File

@ -1848,10 +1848,10 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) {
pOptions->withArbitrator = -1;
pOptions->encryptAlgorithm = -1;
pOptions->dnodeListStr[0] = 0;
pOptions->compactInterval = TSDB_DEFAULT_COMPACT_INTERVAL;
pOptions->compactStartTime = TSDB_DEFAULT_COMPACT_START_TIME;
pOptions->compactEndTime = TSDB_DEFAULT_COMPACT_END_TIME;
pOptions->compactTimeOffset = TSDB_DEFAULT_COMPACT_TIME_OFFSET;
pOptions->compactInterval = -1;
pOptions->compactStartTime = -1;
pOptions->compactEndTime = -1;
pOptions->compactTimeOffset = -1;
return (SNode*)pOptions;
_err:
return NULL;
@ -2033,6 +2033,7 @@ SNode* setAlterDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOp
switch (pAlterOption->type) {
case DB_OPTION_KEEP:
case DB_OPTION_RETENTIONS:
case DB_OPTION_COMPACT_TIME_RANGE:
return setDatabaseOptionImpl(pCxt, pOptions, pAlterOption->type, pAlterOption->pList, true);
default:
break;

22
source/libs/parser/src/parTranslater.c
View File

@ -7989,26 +7989,28 @@ static int32_t checkDbCompactTimeRangeOption(STranslateContext* pCxt, SDatabaseO
pEnd->unit, TIME_UNIT_MINUTE, TIME_UNIT_HOUR, TIME_UNIT_DAY);
}
} else {
pStart->datum.i *= 1440;
pEnd->datum.i *= 1440;
}
pOptions->compactStartTime = getBigintFromValueNode(pStart);
pOptions->compactEndTime = getBigintFromValueNode(pEnd);
if (pOptions->compactStartTime >= pOptions->compactEndTime) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option compact_time_range, start time should be less than end time");
return generateSyntaxErrMsgExt(
&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option compact_time_range: %dm,%dm, start time should be less than end time",
pOptions->compactStartTime, pOptions->compactEndTime);
}
if (pOptions->compactStartTime < -pOptions->keep[2] || pOptions->compactStartTime > -pOptions->daysPerFile) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option compact_time_range, start_time should be in range: [%" PRIi64
"m, %" PRId64 "m]",
-pOptions->keep[2], -pOptions->daysPerFile);
"Invalid option compact_time_range: %dm, start_time should be in range: [%" PRIi64
"m, %dm]",
pOptions->compactStartTime, -pOptions->keep[2], -pOptions->daysPerFile);
}
if (pOptions->compactEndTime < -pOptions->keep[2] || pOptions->compactEndTime > -pOptions->daysPerFile) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option compact_time_range, end time should be in range: [%" PRIi64
"m, %" PRId64 "m]",
-pOptions->keep[2], -pOptions->daysPerFile);
"Invalid option compact_time_range: %dm, end time should be in range: [%" PRIi64
"m, %dm]",
pOptions->compactEndTime, -pOptions->keep[2], -pOptions->daysPerFile);
}
return TSDB_CODE_SUCCESS;
@ -8019,7 +8021,7 @@ static int32_t checkDbCompactTimeOffsetOption(STranslateContext* pCxt, SDatabase
pOptions->compactTimeOffset > TSDB_MAX_COMPACT_TIME_OFFSET) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option compact_time_offset: %d"
" valid range: [%d, %d]",
", valid range: [%d, %d]",
pOptions->compactTimeOffset, TSDB_MIN_COMPACT_TIME_OFFSET,
TSDB_MAX_COMPACT_TIME_OFFSET);
}