diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 3ae4e9941e..fdbb5829f8 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -284,3 +284,5 @@ keepColumnName 1 # 0 no query allowed, queries are disabled # queryBufferSize -1 +# percent of redundant data in tsdb meta will compact meta data,0 means donot compact +# tsdbMetaCompactRatio 30 diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 44b3e87e7d..6c51ba22a1 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -150,6 +150,7 @@ int32_t tsMaxVgroupsPerDb = 0; int32_t tsMinTablePerVnode = TSDB_TABLES_STEP; int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES; int32_t tsTableIncStepPerVnode = TSDB_TABLES_STEP; +int32_t tsTsdbMetaCompactRatio = TSDB_META_COMPACT_RATIO; // balance int8_t tsEnableBalance = 1; @@ -1576,6 +1577,16 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + cfg.option = "tsdbMetaCompactRatio"; + cfg.ptr = &tsTsdbMetaCompactRatio; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; + cfg.minValue = 0; + cfg.maxValue = 100; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + assert(tsGlobalConfigNum <= TSDB_CFG_MAX_NUM); #ifdef TD_TSZ // lossy compress diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index fceeaea0ae..66811326ee 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -275,6 +275,7 @@ do { \ #define TSDB_MAX_TABLES 10000000 #define TSDB_DEFAULT_TABLES 1000000 #define TSDB_TABLES_STEP 1000 +#define TSDB_META_COMPACT_RATIO 30 #define TSDB_MIN_DAYS_PER_FILE 1 #define TSDB_MAX_DAYS_PER_FILE 3650 diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 9e441c1770..6bd47247a2 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -14,6 +14,8 @@ */ #include "tsdbint.h" +extern int32_t tsTsdbMetaCompactRatio; + #define TSDB_MAX_SUBBLOCKS 8 static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) { if (key < 0) { @@ -339,7 +341,7 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) { tsdbCloseMFile(&mf); tsdbUpdateMFile(pfs, &mf); - if (tsdbCompactMetaFile(pRepo, pfs, &mf) < 0) { + if (tsTsdbMetaCompactRatio > 0 && tsdbCompactMetaFile(pRepo, pfs, &mf) < 0) { tsdbError("compact meta file error"); } @@ -455,8 +457,9 @@ static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) { static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) { float delPercent = (float)(pMFile->info.nDels) / (float)(pMFile->info.nRecords); float tombPercent = (float)(pMFile->info.tombSize) / (float)(pMFile->info.size); + float compactRatio = (float)(tsTsdbMetaCompactRatio)/100; - if (delPercent < 0.33 && tombPercent < 0.33) { + if (delPercent < compactRatio && tombPercent < compactRatio) { return 0; } @@ -465,8 +468,8 @@ static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) { return -1; } - tsdbInfo("begin compact tsdb meta file, nDels:%" PRId64 ",nRecords:%" PRId64 ",tombSize:%" PRId64 ",size:%" PRId64, - pMFile->info.nDels,pMFile->info.nRecords,pMFile->info.tombSize,pMFile->info.size); + tsdbInfo("begin compact tsdb meta file, ratio:%d, nDels:%" PRId64 ",nRecords:%" PRId64 ",tombSize:%" PRId64 ",size:%" PRId64, + tsTsdbMetaCompactRatio, pMFile->info.nDels,pMFile->info.nRecords,pMFile->info.tombSize,pMFile->info.size); SMFile mf; SDiskID did;