From 6763ac91101d26a2ec9c1844426722fc9f79322d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 18 Mar 2024 09:12:53 +0800 Subject: [PATCH 1/5] fix(stream):adjust current offset by using the scanning version, not the processed version in WAL. --- source/libs/stream/src/streamMeta.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index a072ee1f6f..5f6440c06d 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1128,7 +1128,11 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { } if ((*pTask)->exec.pWalReader != NULL) { - entry.processedVer = (*pTask)->chkInfo.nextProcessVer - 1; + entry.processedVer = walReaderGetCurrentVer((*pTask)->exec.pWalReader) - 1; + if (entry.processedVer < 0) { + entry.processedVer = (*pTask)->chkInfo.processedVer; + } + walReaderValidVersionRange((*pTask)->exec.pWalReader, &entry.verStart, &entry.verEnd); } From 2b706f00ad2a6d4e49ecf06471cce910ac99c575 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 18 Mar 2024 14:16:49 +0800 Subject: [PATCH 2/5] refactor(meta): add some logs. --- source/dnode/vnode/src/meta/metaOpen.c | 28 ++++++++++++-------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 8055d6e139..0804dacf65 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -246,32 +246,30 @@ int metaAlterCache(SMeta *pMeta, int32_t nPage) { } int32_t metaRLock(SMeta *pMeta) { - int32_t ret = 0; - - metaTrace("meta rlock %p", &pMeta->lock); - - ret = taosThreadRwlockRdlock(&pMeta->lock); + metaTrace("try meta rlock %p, __readers:%d", &pMeta->lock, pMeta->lock.__data.__readers); + int32_t ret = taosThreadRwlockRdlock(&pMeta->lock); + metaTrace("meta rlock %p, code:%d, __readers:%d", &pMeta->lock, ret, pMeta->lock.__data.__readers); return ret; } int32_t metaWLock(SMeta *pMeta) { - int32_t ret = 0; - - metaTrace("meta wlock %p", &pMeta->lock); - - ret = taosThreadRwlockWrlock(&pMeta->lock); + metaTrace("try meta wlock %p, __readers:%d, __cur_writer:%x", &pMeta->lock, pMeta->lock.__data.__readers, + pMeta->lock.__data.__cur_writer); + int32_t ret = taosThreadRwlockWrlock(&pMeta->lock); + metaTrace("meta wlock %p completed, code:%d, __readers:%d, __cur_writer:%x", &pMeta->lock, ret, + pMeta->lock.__data.__readers, pMeta->lock.__data.__cur_writer); return ret; } int32_t metaULock(SMeta *pMeta) { - int32_t ret = 0; - - metaTrace("meta ulock %p", &pMeta->lock); - - ret = taosThreadRwlockUnlock(&pMeta->lock); + metaTrace("try meta ulock %p, __readers:%d, __cur_writer:%x", &pMeta->lock, pMeta->lock.__data.__readers, + pMeta->lock.__data.__cur_writer); + int32_t ret = taosThreadRwlockUnlock(&pMeta->lock); + metaTrace("meta ulock %p, code:%d, __readers:%d, __cur_writer:%x", &pMeta->lock, ret, pMeta->lock.__data.__readers, + pMeta->lock.__data.__cur_writer); return ret; } From 6795f6e1ccda82bc95b45308ea81cfc39ec58630 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 19 Mar 2024 15:16:46 +0800 Subject: [PATCH 3/5] fix(tsdb): handle the case that no stt statistics info existsin stt files generated by early version of TDengine. --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 4 ++-- source/dnode/vnode/src/tsdb/tsdbRead2.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 0b86cae1be..ba0c5697c0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -429,7 +429,7 @@ static int32_t doLoadSttFilesBlk(SSttBlockLoadInfo *pBlockLoadInfo, SLDataIter * code = loadTombFn(pReader1, pIter->pReader, pIter->pBlockLoadInfo); double el = (taosGetTimestampUs() - st) / 1000.0; - tsdbDebug("load the stt file info completed, elapsed time:%.2fms, %s", el, idStr); + tsdbDebug("load the stt file blk info completed, elapsed time:%.2fms, %s", el, idStr); return code; } @@ -806,7 +806,7 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoF tMergeTreeAddIter(pMTree, pIter); // let's record the time window for current table of uid in the stt files - if (pSttDataInfo != NULL) { + if (pSttDataInfo != NULL && numOfRows > 0) { taosArrayPush(pSttDataInfo->pTimeWindowList, &w); pSttDataInfo->numOfRows += numOfRows; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index c0ad32d269..69f4f82459 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -2093,7 +2093,7 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan pScanInfo->sttKeyInfo.status = taosArrayGetSize(info.pTimeWindowList) ? STT_FILE_HAS_DATA : STT_FILE_NO_DATA; pScanInfo->sttKeyInfo.nextProcKey = ASCENDING_TRAVERSE(pReader->info.order) ? pScanInfo->sttWindow.skey : pScanInfo->sttWindow.ekey; - hasData = true; + hasData = (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA); } else { // not clean stt blocks INIT_TIMEWINDOW(&pScanInfo->sttWindow); //reset the time window pScanInfo->sttBlockReturned = false; From 1c2c2b2b9aeef81b43bd9094e18018f316a7eb8d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 19 Mar 2024 15:20:38 +0800 Subject: [PATCH 4/5] refactor: do some internal refactor. --- source/dnode/vnode/src/meta/metaOpen.c | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 0804dacf65..522bfeaf2c 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -246,30 +246,20 @@ int metaAlterCache(SMeta *pMeta, int32_t nPage) { } int32_t metaRLock(SMeta *pMeta) { - metaTrace("try meta rlock %p, __readers:%d", &pMeta->lock, pMeta->lock.__data.__readers); + metaTrace("try meta rlock %p", &pMeta->lock); int32_t ret = taosThreadRwlockRdlock(&pMeta->lock); - - metaTrace("meta rlock %p, code:%d, __readers:%d", &pMeta->lock, ret, pMeta->lock.__data.__readers); return ret; } int32_t metaWLock(SMeta *pMeta) { - metaTrace("try meta wlock %p, __readers:%d, __cur_writer:%x", &pMeta->lock, pMeta->lock.__data.__readers, - pMeta->lock.__data.__cur_writer); - + metaTrace("meta wlock %p", &pMeta->lock); int32_t ret = taosThreadRwlockWrlock(&pMeta->lock); - metaTrace("meta wlock %p completed, code:%d, __readers:%d, __cur_writer:%x", &pMeta->lock, ret, - pMeta->lock.__data.__readers, pMeta->lock.__data.__cur_writer); return ret; } int32_t metaULock(SMeta *pMeta) { - metaTrace("try meta ulock %p, __readers:%d, __cur_writer:%x", &pMeta->lock, pMeta->lock.__data.__readers, - pMeta->lock.__data.__cur_writer); - + metaTrace("meta ulock %p", &pMeta->lock); int32_t ret = taosThreadRwlockUnlock(&pMeta->lock); - metaTrace("meta ulock %p, code:%d, __readers:%d, __cur_writer:%x", &pMeta->lock, ret, pMeta->lock.__data.__readers, - pMeta->lock.__data.__cur_writer); return ret; } From 77f8dbeeb99c3ce3fff117470e1dde8f30dbe9a0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 19 Mar 2024 15:21:23 +0800 Subject: [PATCH 5/5] refactor: do some internal refactor. --- source/dnode/vnode/src/meta/metaOpen.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 522bfeaf2c..4a3dcd0e51 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -246,7 +246,7 @@ int metaAlterCache(SMeta *pMeta, int32_t nPage) { } int32_t metaRLock(SMeta *pMeta) { - metaTrace("try meta rlock %p", &pMeta->lock); + metaTrace("meta rlock %p", &pMeta->lock); int32_t ret = taosThreadRwlockRdlock(&pMeta->lock); return ret; }