From 580b7124cf1a8c35fd0736712e40b9ef58bbf990 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 27 Jul 2023 09:16:28 +0800 Subject: [PATCH 1/5] fix(tsdb/commit2): new maxDelKey for commit context --- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index ed05d7a6ca..24427d1688 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -37,6 +37,7 @@ typedef struct { int64_t cid; int64_t now; TSKEY nextKey; + TSKEY maxDelKey; int32_t fid; int32_t expLevel; SDiskID did; @@ -161,6 +162,9 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) { SMetaInfo info; if (committer->ctx->fset == NULL && !committer->ctx->hasTSData) { + if (committer->ctx->maxKey < committer->ctx->maxDelKey) { + committer->ctx->nextKey = committer->ctx->maxKey + 1; + } return 0; } @@ -185,16 +189,18 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) { goto _next; } + TSKEY maxKey = committer->ctx->maxKey; if (record->ekey > committer->ctx->maxKey) { - committer->ctx->maxKey = committer->ctx->maxKey + 1; + maxKey = committer->ctx->maxKey + 1; } - if (record->ekey > committer->ctx->maxKey) { - committer->ctx->nextKey = record->ekey; + if (record->ekey > committer->ctx->maxKey && committer->ctx->nextKey > maxKey) { + committer->ctx->nextKey = maxKey; + committer->ctx->maxDelKey = TMAX(record->ekey, committer->ctx->maxDelKey); } record->skey = TMAX(record->skey, committer->ctx->minKey); - record->ekey = TMIN(record->ekey, committer->ctx->maxKey); + record->ekey = TMIN(record->ekey, maxKey); numRecord++; code = tsdbFSetWriteTombRecord(committer->writer, record); From 992d623ae5c16e637d4c65b57fb92780e0897ee9 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 27 Jul 2023 09:34:47 +0800 Subject: [PATCH 2/5] fix(tsdb/commit2): initialize ctx's maxDelKey with TSKEY_MIN --- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 24427d1688..49e25f5fbb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -467,6 +467,7 @@ static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *co committer->compactVersion = INT64_MAX; committer->ctx->cid = tsdbFSAllocEid(tsdb->pFS); committer->ctx->now = taosGetTimestampSec(); + committer->ctx->maxDelKey = TSKEY_MIN; committer->ctx->nextKey = tsdb->imem->minKey; if (tsdb->imem->nDel > 0) { From dd32f12f92b9d232e3de1361f36ed370e1144f37 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 27 Jul 2023 09:50:32 +0800 Subject: [PATCH 3/5] tsdb/commit2: reset nextKey to max to stop commit --- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 49e25f5fbb..1bdbe54231 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -164,6 +164,8 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) { if (committer->ctx->fset == NULL && !committer->ctx->hasTSData) { if (committer->ctx->maxKey < committer->ctx->maxDelKey) { committer->ctx->nextKey = committer->ctx->maxKey + 1; + } else { + committer->ctx->nextKey = TSKEY_MAX; } return 0; } From 8a2764c698e68ec8d4d19751bd39b01450a2eb4f Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 27 Jul 2023 10:35:18 +0800 Subject: [PATCH 4/5] tsdb/commit2: use max fileset's maxKey as initial maxDelKey --- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 1bdbe54231..6b0ee92252 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -198,7 +198,9 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) { if (record->ekey > committer->ctx->maxKey && committer->ctx->nextKey > maxKey) { committer->ctx->nextKey = maxKey; - committer->ctx->maxDelKey = TMAX(record->ekey, committer->ctx->maxDelKey); + if (record->ekey < TSKEY_MAX) { + committer->ctx->maxDelKey = record->ekey; + } } record->skey = TMAX(record->skey, committer->ctx->minKey); @@ -469,7 +471,13 @@ static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *co committer->compactVersion = INT64_MAX; committer->ctx->cid = tsdbFSAllocEid(tsdb->pFS); committer->ctx->now = taosGetTimestampSec(); + committer->ctx->maxDelKey = TSKEY_MIN; + if (TARRAY2_SIZE(committer->fsetArr) > 0) { + STFileSet *fset = TARRAY2_LAST(committer->fsetArr); + TSKEY minKey; + tsdbFidKeyRange(fset->fid, committer->minutes, committer->precision, &minKey, &committer->ctx->maxDelKey); + } committer->ctx->nextKey = tsdb->imem->minKey; if (tsdb->imem->nDel > 0) { From fe495f701342f64a277595f46acfbe663f1d1a87 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 27 Jul 2023 12:44:36 +0800 Subject: [PATCH 5/5] tsdb/commit2: fix commit minKey for del --- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 25 ++++++++++++++--------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 6b0ee92252..0639cd91a5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -198,9 +198,6 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) { if (record->ekey > committer->ctx->maxKey && committer->ctx->nextKey > maxKey) { committer->ctx->nextKey = maxKey; - if (record->ekey < TSKEY_MAX) { - committer->ctx->maxDelKey = record->ekey; - } } record->skey = TMAX(record->skey, committer->ctx->minKey); @@ -472,13 +469,6 @@ static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *co committer->ctx->cid = tsdbFSAllocEid(tsdb->pFS); committer->ctx->now = taosGetTimestampSec(); - committer->ctx->maxDelKey = TSKEY_MIN; - if (TARRAY2_SIZE(committer->fsetArr) > 0) { - STFileSet *fset = TARRAY2_LAST(committer->fsetArr); - TSKEY minKey; - tsdbFidKeyRange(fset->fid, committer->minutes, committer->precision, &minKey, &committer->ctx->maxDelKey); - } - committer->ctx->nextKey = tsdb->imem->minKey; if (tsdb->imem->nDel > 0) { SRBTreeIter iter[1] = {tRBTreeIterCreate(tsdb->imem->tbDataTree, 1)}; @@ -494,6 +484,21 @@ static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *co } } + committer->ctx->maxDelKey = TSKEY_MIN; + TSKEY minKey = TSKEY_MAX; + TSKEY maxKey = TSKEY_MIN; + if (TARRAY2_SIZE(committer->fsetArr) > 0) { + STFileSet *fset = TARRAY2_LAST(committer->fsetArr); + tsdbFidKeyRange(fset->fid, committer->minutes, committer->precision, &minKey, &committer->ctx->maxDelKey); + + fset = TARRAY2_FIRST(committer->fsetArr); + tsdbFidKeyRange(fset->fid, committer->minutes, committer->precision, &minKey, &maxKey); + } + + if (committer->ctx->nextKey < TMIN(tsdb->imem->minKey, minKey)) { + committer->ctx->nextKey = TMIN(tsdb->imem->minKey, minKey); + } + _exit: if (code) { TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);