From e66a1d6c9aa51aa71a9b58ffa0699813f91a5559 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 29 Nov 2023 09:14:44 +0800 Subject: [PATCH 1/2] enh: limit snap replication msg by size of block data in tsdbSnapReadTimeSeriesData --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index a9da0fbcec..8a7d1f8b97 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -278,6 +278,15 @@ _exit: return code; } +static int64_t tBlockDataSize(SBlockData* pBlockData) { + int64_t nData = 0; + for (int32_t iCol = 0; iCol < pBlockData->nColData; iCol++) { + SColData* pColData = tBlockDataGetColDataByIdx(pBlockData, iCol); + nData += pColData->nData; + } + return nData; +} + static int32_t tsdbSnapReadTimeSeriesData(STsdbSnapReader* reader, uint8_t** data) { int32_t code = 0; int32_t lino = 0; @@ -320,8 +329,11 @@ static int32_t tsdbSnapReadTimeSeriesData(STsdbSnapReader* reader, uint8_t** dat code = tsdbIterMergerNext(reader->dataIterMerger); TSDB_CHECK_CODE(code, lino, _exit); - if (reader->blockData->nRow >= 81920) { - break; + if (!(reader->blockData->nRow % 16)) { + int64_t nData = tBlockDataSize(reader->blockData); + if (nData >= 1 * 1024 * 1024) { + break; + } } } From 9e8944d17cf9d7433b956a5073766c1125908bae Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 29 Nov 2023 09:18:05 +0800 Subject: [PATCH 2/2] enh: adjust buffer size of sync snap replication --- include/util/tdef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 69d0c1126d..1a440c7268 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -305,7 +305,7 @@ typedef enum ELogicConditionType { #define TSDB_SYNC_APPLYQ_SIZE_LIMIT 512 #define TSDB_SYNC_NEGOTIATION_WIN 512 -#define TSDB_SYNC_SNAP_BUFFER_SIZE 2048 +#define TSDB_SYNC_SNAP_BUFFER_SIZE 1024 #define TSDB_TBNAME_COLUMN_INDEX (-1) #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta