From b03baaed2cb7ce442941d846437fcb8bab00f00e Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 30 Oct 2023 19:49:50 +0800 Subject: [PATCH 1/3] chore: revert the code change --- source/dnode/vnode/src/inc/sma.h | 2 +- source/dnode/vnode/src/sma/smaCommit.c | 2 +- source/dnode/vnode/src/sma/smaRollup.c | 4 +--- tests/script/tsim/sma/rsmaPersistenceRecovery.sim | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index a029aadc0b..aaf0973b41 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -218,7 +218,7 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree); int32_t tdRSmaRestore(SSma *pSma, int8_t type, int64_t committedVer, int8_t rollback); int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName); int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type); -int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash); +// int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash); int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer, int8_t rollback); void tdRSmaQTaskInfoGetFullPath(SVnode *pVnode, tb_uid_t suid, int8_t level, STfs *pTfs, char *outputName); diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 652aab3c01..c26157f4b7 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -178,7 +178,7 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma, bool isCommit) { if (!isCommit) goto _exit; - code = tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)); + // code = tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)); TSDB_CHECK_CODE(code, lino, _exit); smaInfo("vgId:%d, rsma commit, operator state committed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 0d9ff4262f..8da2fff5a6 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -1052,14 +1052,13 @@ _err: return code; } -#if 1 +#if 0 int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { int32_t code = 0; int32_t lino = 0; SSma *pSma = pRSmaStat->pSma; SVnode *pVnode = pSma->pVnode; SRSmaFS fs = {0}; - smaInfo("prop:%s:%d start", __func__, __LINE__); if (taosHashGetSize(pInfoHash) <= 0) { return TSDB_CODE_SUCCESS; @@ -1076,7 +1075,6 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pRSmaInfo, i); if (pItem && pItem->pStreamState) { - smaInfo("prop:%s:%d streamState commit", __func__, __LINE__); if (streamStateCommit(pItem->pStreamState) < 0) { code = TSDB_CODE_RSMA_STREAM_STATE_COMMIT; TSDB_CHECK_CODE(code, lino, _exit); diff --git a/tests/script/tsim/sma/rsmaPersistenceRecovery.sim b/tests/script/tsim/sma/rsmaPersistenceRecovery.sim index c70f2dc20a..6f78829db7 100644 --- a/tests/script/tsim/sma/rsmaPersistenceRecovery.sim +++ b/tests/script/tsim/sma/rsmaPersistenceRecovery.sim @@ -5,7 +5,7 @@ sleep 50 sql connect #todo wait for streamState checkpoint -#return 1 +return 1 print =============== create database with retentions sql create database d0 retentions -:7d,5m:21d,15m:365d; From 28cd8f4915e15f50439f09e4e7ed7088dde3935f Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 31 Oct 2023 11:38:33 +0800 Subject: [PATCH 2/3] enh: file duration for rsma --- include/util/tdef.h | 2 +- source/dnode/vnode/src/sma/smaOpen.c | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index bb5d4cfa96..21a290d358 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -325,7 +325,7 @@ typedef enum ELogicConditionType { #define TSDB_MAX_DAYS_PER_FILE (3650 * 1440) #define TSDB_DEFAULT_DAYS_PER_FILE (10 * 1440) #define TSDB_MIN_DURATION_PER_FILE 60 // unit minute -#define TSDB_MAX_DURATION_PER_FILE (3650 * 1440) +#define TSDB_MAX_DURATION_PER_FILE (90 * 1440) #define TSDB_DEFAULT_DURATION_PER_FILE (10 * 1440) #define TSDB_MIN_KEEP (1 * 1440) // data in db to be reserved. unit minute #define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved. diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index f0e329dfd7..633e096314 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -79,20 +79,18 @@ static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t p freqDuration = convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE); keepDuration = convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE); - int32_t nFreqTimes = (r + level)->freq / (10 * 1000); // use 10s for freq of 1st level + int32_t nFreqTimes = (r + level)->freq / (60 * 1000); // use 60s for freq of 1st level days *= (nFreqTimes > 1 ? nFreqTimes : 1); - if (days > keepDuration) { - days = keepDuration; - } - - if (days > TSDB_MAX_DURATION_PER_FILE) { - days = TSDB_MAX_DURATION_PER_FILE; - } - if (days < freqDuration) { days = freqDuration; } + + int32_t maxKeepDuration = TMIN(keepDuration, TSDB_MAX_DURATION_PER_FILE); + if (days > maxKeepDuration) { + days = maxKeepDuration; + } + _exit: smaInfo("vgId:%d, evaluated duration for level %d is %d, raw val:%d", TD_VID(pVnode), level + 1, days, duration); return days; From 48d5a908c20ba2059e6dd9eadbc31b09dfaff60a Mon Sep 17 00:00:00 2001 From: haoranchen Date: Tue, 31 Oct 2023 18:14:53 +0800 Subject: [PATCH 3/3] increase the timeout for CI --- Jenkinsfile2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 165d203a22..f308db759d 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -424,7 +424,7 @@ pipeline { echo "${WKDIR}/restore.sh -p ${BRANCH_NAME} -n ${BUILD_ID} -c {container name}" } catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { - timeout(time: 130, unit: 'MINUTES'){ + timeout(time: 150, unit: 'MINUTES'){ pre_test() script { sh '''