xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

enh: use placeholder '-' for interval of 1st retention level

This commit is contained in:
kailixu 2023-10-30 14:02:31 +08:00
parent 53e9c483c9
commit c77153cb81
14 changed files with 2687 additions and 3902 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/common/tmsg.h
View File

@ -451,7 +451,7 @@ typedef struct SRetention {
int8_t keepUnit;
} SRetention;
#define RETENTION_VALID(r) (((r)->freq > 0) && ((r)->keep > 0))
#define RETENTION_VALID(l, r) ((((l) == 0 && (r)->freq >= 0) || ((r)->freq > 0)) && ((r)->keep > 0))
#pragma pack(push, 1)

2
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -137,7 +137,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
SRetention *pRetention = &pCfg->tsdbCfg.retentions[i];
memcpy(pRetention, taosArrayGet(pCreate->pRetensions, i), sizeof(SRetention));
if (i == 0) {
if ((pRetention->freq > 0 && pRetention->keep > 0)) pCfg->isRsma = 1;
if ((pRetention->freq >= 0 && pRetention->keep > 0)) pCfg->isRsma = 1;
}
}

31
source/dnode/vnode/src/sma/smaOpen.c
View File

@ -31,21 +31,21 @@ static int32_t rsmaRestore(SSma *pSma);
} while (0)
#define SMA_OPEN_RSMA_IMPL(v, l, force) \
do { \
SRetention *r = (SRetention *)VND_RETENTIONS(v) + l; \
if (!RETENTION_VALID(r)) { \
if (l == 0) { \
code = TSDB_CODE_INVALID_PARA; \
TSDB_CHECK_CODE(code, lino, _exit); \
} \
break; \
} \
code = smaSetKeepCfg(v, &keepCfg, pCfg, TSDB_TYPE_RSMA_L##l); \
TSDB_CHECK_CODE(code, lino, _exit); \
do { \
SRetention *r = (SRetention *)VND_RETENTIONS(v) + l; \
if (!RETENTION_VALID(l, r)) { \
if (l == 0) { \
code = TSDB_CODE_INVALID_PARA; \
TSDB_CHECK_CODE(code, lino, _exit); \
} \
break; \
} \
code = smaSetKeepCfg(v, &keepCfg, pCfg, TSDB_TYPE_RSMA_L##l); \
TSDB_CHECK_CODE(code, lino, _exit); \
if (tsdbOpen(v, &SMA_RSMA_TSDB##l(pSma), VNODE_RSMA##l##_DIR, &keepCfg, rollback, force) < 0) { \
code = terrno; \
TSDB_CHECK_CODE(code, lino, _exit); \
} \
code = terrno; \
TSDB_CHECK_CODE(code, lino, _exit); \
} \
} while (0)
/**
@ -79,7 +79,7 @@ static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t p
freqDuration = convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE);
keepDuration = convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE);
int32_t nFreqTimes = (r + level)->freq / (r + TSDB_RETENTION_L0)->freq;
int32_t nFreqTimes = (r + level)->freq / (10 * 1000); // use 10s for freq of 1st level
days *= (nFreqTimes > 1 ? nFreqTimes : 1);
if (days > keepDuration) {
@ -157,6 +157,7 @@ int32_t smaOpen(SVnode *pVnode, int8_t rollback, bool force) {
_exit:
if (code) {
smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code));
terrno = code;
}
return code;
}

2
source/dnode/vnode/src/vnd/vnodeCfg.c
View File

@ -106,7 +106,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset) < 0) return -1;
if (pCfg->tsdbCfg.retentions[0].freq > 0) {
if (pCfg->tsdbCfg.retentions[0].keep > 0) {
int32_t nRetention = 1;
if (pCfg->tsdbCfg.retentions[1].freq > 0) {
++nRetention;

37
source/libs/command/src/command.c
View File

@ -215,37 +215,28 @@ int64_t getValOfDiffPrecision(int8_t unit, int64_t val) {
return v;
}
char* buildRetension(SArray* pRetension) {
static char* buildRetension(SArray* pRetension) {
size_t size = taosArrayGetSize(pRetension);
if (size == 0) {
return NULL;
}
char* p1 = taosMemoryCalloc(1, 100);
SRetention* p = taosArrayGet(pRetension, 0);
char* p1 = taosMemoryCalloc(1, 100);
int32_t len = 0;
int64_t v1 = getValOfDiffPrecision(p->freqUnit, p->freq);
int64_t v2 = getValOfDiffPrecision(p->keepUnit, p->keep);
len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit);
for (int32_t i = 0; i < size; ++i) {
SRetention* p = TARRAY_GET_ELEM(pRetension, i);
int64_t v1 = getValOfDiffPrecision(p->freqUnit, p->freq);
int64_t v2 = getValOfDiffPrecision(p->keepUnit, p->keep);
if (i == 0) {
len += sprintf(p1 + len, "-:%" PRId64 "%c", v2, p->keepUnit);
} else {
len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit);
}
if (size > 1) {
len += sprintf(p1 + len, ",");
p = taosArrayGet(pRetension, 1);
v1 = getValOfDiffPrecision(p->freqUnit, p->freq);
v2 = getValOfDiffPrecision(p->keepUnit, p->keep);
len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit);
}
if (size > 2) {
len += sprintf(p1 + len, ",");
p = taosArrayGet(pRetension, 2);
v1 = getValOfDiffPrecision(p->freqUnit, p->freq);
v2 = getValOfDiffPrecision(p->keepUnit, p->keep);
len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit);
if (i < size - 1) {
len += sprintf(p1 + len, ",");
}
}
return p1;

1
source/libs/parser/inc/sql.y
View File

@ -304,6 +304,7 @@ retention_list(A) ::= retention(B).
retention_list(A) ::= retention_list(B) NK_COMMA retention(C). { A = addNodeToList(pCxt, B, C); }
retention(A) ::= NK_VARIABLE(B) NK_COLON NK_VARIABLE(C). { A = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &B), createDurationValueNode(pCxt, &C)); }
retention(A) ::= NK_MINUS(B) NK_COLON NK_VARIABLE(C). { A = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &B), createDurationValueNode(pCxt, &C)); }
%type speed_opt { int32_t }
%destructor speed_opt { }

14
source/libs/parser/src/parTranslater.c
View File

@ -4588,10 +4588,22 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete
SValueNode* pPrevFreq = NULL;
SValueNode* pPrevKeep = NULL;
SNode* pRetention = NULL;
bool firstFreq = true;
FOREACH(pRetention, pRetentions) {
SNode* pNode = NULL;
FOREACH(pNode, ((SNodeListNode*)pRetention)->pNodeList) {
SValueNode* pVal = (SValueNode*)pNode;
if (firstFreq) {
firstFreq = false;
if (pVal->literal[0] != '-' || strlen(pVal->literal) != 1) {
return generateSyntaxErrMsgExt(
&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option retentions(freq): %s, the interval of 1st retention level should be '-'", pVal->literal);
}
pVal->unit = TIME_UNIT_SECOND; // assign minimum unit
pVal->datum.i = 0; // assign minimum value
continue;
}
if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
return pCxt->errCode;
}
@ -4616,7 +4628,7 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete
}
// check value range
if (pFreq->datum.i <= 0) {
if (pPrevFreq != NULL && pFreq->datum.i <= 0) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option retentions(freq): %s should larger than 0", pFreq->literal);
}

6404
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

2
tests/script/tsim/sma/rsmaCreateInsertQuery.sim
View File

@ -5,7 +5,7 @@ sleep 50
sql connect
print =============== create database with retentions
sql create database d0 retentions 5s:7d,10s:21d,15s:365d;
sql create database d0 retentions -:7d,10s:21d,15s:365d;
sql use d0
print =============== create super table and register rsma

2
tests/script/tsim/sma/rsmaPersistenceRecovery.sim
View File

@ -8,7 +8,7 @@ sql connect
return 1
print =============== create database with retentions
sql create database d0 retentions 5s:7d,5m:21d,15m:365d;
sql create database d0 retentions -:7d,5m:21d,15m:365d;
sql use d0
print =============== create super table and register rsma

2
tests/script/tsim/sync/vnodesnapshot-rsma-test.sim
View File

@ -47,7 +47,7 @@ endi
$replica = 3
$vgroups = 1
$retentions = 5s:7d,15s:21d,1m:365d
$retentions = -:7d,15s:21d,1m:365d
print ============= create database
sql create database db replica $replica vgroups $vgroups retentions $retentions

4
tests/system-test/1-insert/block_wise.py
View File

@ -417,7 +417,7 @@ class TDTestCase:
self.all_test()
tdLog.printNoPrefix("==========step2:create table in rollup database")
tdSql.execute("create database db3 retentions 1s:4m,2s:8m,3s:12m")
tdSql.execute("create database db3 retentions -:4m,2s:8m,3s:12m")
tdSql.execute("use db3")
tdSql.query(f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(first) watermark 5s max_delay 1m sma({INT_COL})")
@ -438,7 +438,7 @@ class TDTestCase:
tdSql.execute("drop database if exists db_s20 ")
tdLog.printNoPrefix("==========step4:insert and flush in rollup database")
tdSql.execute("create database db4 retentions 1s:4m,2s:8m,3s:12m")
tdSql.execute("create database db4 retentions -:4m,2s:8m,3s:12m")
tdSql.execute("use db4")
self.__create_tb(rollup="first")
self.__insert_data(rollup="first")

82
tests/system-test/1-insert/create_retentions.py
View File

@ -54,53 +54,63 @@ class TDTestCase:
return [
# check grammar
"create database db1 retentions",
"create database db1 retentions 1s:1d,2s:2d,3s:3d,4s:4d",
"create database db1 retentions 1s:1d",
"create database db1 retentions 1s:1d,2s:2d",
"create database db1 retentions 1s:1d,2s:2d,3s:3d",
"create database db1 retentions -:1d,2s:2d,3s:3d,4s:4d",
"create database db1 retentions -:-",
"create database db1 retentions --:1d",
"create database db1 retentions +:1d",
"create database db1 retentions :1d",
"create database db1 retentions -:1d,-:2d",
"create database db1 retentions -:1d,1s:-",
"create database db1 retentions -:1d,15s:2d,-:3d",
# check unit
"create database db1 retentions 1b:1d",
"create database db1 retentions 1u:1d",
"create database db1 retentions 1a:1d",
"create database db1 retentions 1n:1d",
"create database db1 retentions 1y:1d",
"create database db1 retentions 1s:86400s",
"create database db1 retentions 1s:86400000a",
"create database db1 retentions 1s:86400000000u",
"create database db1 retentions 1s:86400000000000b",
"create database db1 retentions 1s:1w",
"create database db1 retentions 1s:1n",
"create database db1 retentions 1s:1y",
"create database db1 retentions -:1d,1b:1d",
"create database db1 retentions -:1d,1u:1d",
"create database db1 retentions -:1d,1a:1d",
"create database db1 retentions -:1d,1n:1d",
"create database db1 retentions -:1d,1y:1d",
"create database db1 retentions -:1d,1s:86400s",
"create database db1 retentions -:1d,1s:86400000a",
"create database db1 retentions -:1d,1s:86400000000u",
"create database db1 retentions -:1d,1s:86400000000000b",
"create database db1 retentions -:1d,1s:1w",
"create database db1 retentions -:1d,1s:1n",
"create database db1 retentions -:1d,1s:1y",
# check value range
"create database db1 retentions -1s:1d",
"create database db1 retentions 0s:1d",
"create database db3 retentions 1s:-1d",
"create database db3 retentions 1s:0d",
"create database db3 retentions 1s:1439m",
"create database db3 retentions 1s:365001d",
"create database db3 retentions 1s:8760001h",
"create database db3 retentions 1s:525600001m",
"create database db3 retentions 1s:106581d precision 'ns'",
"create database db3 retentions 1s:2557921h precision 'ns'",
"create database db3 retentions 1s:153475201m precision 'ns'",
"create database db3 retentions -:-1d",
"create database db3 retentions -:0d",
"create database db3 retentions -:1439m",
"create database db3 retentions -:365001d",
"create database db3 retentions -:8760001h",
"create database db3 retentions -:525600001m",
"create database db3 retentions -:106581d precision 'ns'",
"create database db3 retentions -:2557921h precision 'ns'",
"create database db3 retentions -:153475201m precision 'ns'",
# check relationships
"create database db5 retentions 1441m:1440m,2d:3d",
"create database db5 retentions 2m:1d,1s:2d",
"create database db5 retentions 1s:2880m,2s:2879m",
"create database db5 retentions 1s:1d,2s:2d,2s:3d",
"create database db5 retentions 1s:1d,3s:2d,2s:3d",
"create database db1 retentions 1s:1d,2s:3d,3s:2d",
"create database db5 retentions -:1440m,1441m:1440m,2d:3d",
"create database db5 retentions -:1d,2m:1d,1s:2d",
"create database db5 retentions -:1440m,1s:2880m,2s:2879m",
"create database db5 retentions -:1d,2s:2d,2s:3d",
"create database db5 retentions -:1d,3s:2d,2s:3d",
"create database db1 retentions -:1d,2s:3d,3s:2d",
]
@property
def create_databases_sql_current(self):
return [
f"create database {DB1} retentions 1s:1d",
f"create database {DB2} retentions 1s:1d,2m:2d,3h:3d",
f"create database {DB1} retentions -:1d",
f"create database {DB2} retentions -:1d,2m:2d,3h:3d",
]
@property
def alter_database_sql(self):
return [
"alter database db1 retentions 99h:99d",
"alter database db2 retentions 97h:97d,98h:98d,99h:99d,",
"alter database db1 retentions -:99d",
"alter database db2 retentions -:97d,98h:98d,99h:99d,",
]
@property
@ -281,7 +291,7 @@ class TDTestCase:
tdLog.printNoPrefix("==========step2:create table in rollup database")
tdLog.printNoPrefix("==========step2.1 : rolluo func is not last/first")
tdSql.prepare(dbname=DB3, **{"retentions": "1s:1d, 3s:3d, 5s:5d"})
tdSql.prepare(dbname=DB3, **{"retentions": "-:1d, 3s:3d, 5s:5d"})
db3_ctb_num = 10
self.__create_tb(rsma=True, dbname=DB3, ctb_num=db3_ctb_num, stb=STBNAME)
@ -338,7 +348,7 @@ class TDTestCase:
tdLog.printNoPrefix("==========step2.2 : rolluo func is last/first")
tdSql.prepare(dbname=DB4, **{"retentions": "1s:1d, 2m:3d, 3m:5d"})
tdSql.prepare(dbname=DB4, **{"retentions": "-:1d, 2m:3d, 3m:5d"})
db4_ctb_num = 10
tdSql.execute(f"use {DB4}")

4
tests/system-test/1-insert/time_range_wise.py
View File

@ -584,7 +584,7 @@ class TDTestCase:
self.all_test()
tdLog.printNoPrefix("==========step2:create table in rollup database")
tdSql.execute("create database db3 retentions 1s:4m,2s:8m,3s:12m")
tdSql.execute("create database db3 retentions -:4m,2s:8m,3s:12m")
tdSql.execute("use db3")
tdSql.execute(f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(first) watermark 5s max_delay 1m sma({INT_COL}) ")
self.all_test()
@ -603,7 +603,7 @@ class TDTestCase:
# add for TS-2440
for i in range(self.rows):
tdSql.execute("drop database if exists db3 ")
tdSql.execute("create database db3 retentions 1s:4m,2s:8m,3s:12m")
tdSql.execute("create database db3 retentions -:4m,2s:8m,3s:12m")
def stop(self):
tdSql.close()