From 8cbaebc5a6bab201c14689306ecdb60333eea7d8 Mon Sep 17 00:00:00 2001 From: silenceqi Date: Tue, 31 Dec 2024 11:40:56 +0800 Subject: [PATCH] feat: add some index metrics related to segment memory (#65) * feat: add some index metrics related to segment memory * chore: update release notes --- docs/content.en/docs/release-notes/_index.md | 1 + modules/elastic/api/index_metrics.go | 48 +++++++++++++++++ modules/elastic/api/index_overview.go | 7 +++ modules/elastic/api/v1/index_metrics.go | 54 +++++++++++++++++++- modules/elastic/api/v1/index_overview.go | 7 +++ 5 files changed, 116 insertions(+), 1 deletion(-) diff --git a/docs/content.en/docs/release-notes/_index.md b/docs/content.en/docs/release-notes/_index.md index d7e18cac..49a07c9d 100644 --- a/docs/content.en/docs/release-notes/_index.md +++ b/docs/content.en/docs/release-notes/_index.md @@ -13,6 +13,7 @@ Information about release notes of INFINI Console is provided here. ### Features - Add allocation to activities if is cluster health change and changed to red. +- Add index metrics for segment memory (norms, points, version map, fixed bit set). ### Bug fix - Fixed query thread pool metrics when cluster uuid is empty diff --git a/modules/elastic/api/index_metrics.go b/modules/elastic/api/index_metrics.go index 1936af01..b73545d6 100644 --- a/modules/elastic/api/index_metrics.go +++ b/modules/elastic/api/index_metrics.go @@ -607,6 +607,54 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu FormatType: "bytes", Units: "", }) + case v1.SegmentNormsMetricKey: + segmentNormsMetric := newMetricItem(v1.SegmentNormsMetricKey, 17, MemoryGroupKey) + segmentNormsMetric.AddAxi("Segment norms memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: v1.SegmentNormsMetricKey, + Field: "payload.elasticsearch.shard_stats.segments.norms_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentNormsMetric, + FormatType: "bytes", + Units: "", + }) + case v1.SegmentPointsMetricKey: + segmentPointsMetric := newMetricItem(v1.SegmentPointsMetricKey, 18, MemoryGroupKey) + segmentPointsMetric.AddAxi("Segment points memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: v1.SegmentPointsMetricKey, + Field: "payload.elasticsearch.shard_stats.segments.points_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentPointsMetric, + FormatType: "bytes", + Units: "", + }) + case v1.VersionMapMetricKey: + segmentVersionMapMetric := newMetricItem(v1.VersionMapMetricKey, 18, MemoryGroupKey) + segmentVersionMapMetric.AddAxi("Segment version map memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: v1.VersionMapMetricKey, + Field: "payload.elasticsearch.shard_stats.segments.version_map_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentVersionMapMetric, + FormatType: "bytes", + Units: "", + }) + case v1.FixedBitSetMetricKey: + segmentFixedBitSetMetric := newMetricItem(v1.FixedBitSetMetricKey, 18, MemoryGroupKey) + segmentFixedBitSetMetric.AddAxi("Segment fixed bit set memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: v1.FixedBitSetMetricKey, + Field: "payload.elasticsearch.shard_stats.segments.fixed_bit_set_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentFixedBitSetMetric, + FormatType: "bytes", + Units: "", + }) } diff --git a/modules/elastic/api/index_overview.go b/modules/elastic/api/index_overview.go index 2a5d3ecc..6efdc23f 100644 --- a/modules/elastic/api/index_overview.go +++ b/modules/elastic/api/index_overview.go @@ -1018,6 +1018,13 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ return value / value2 } metricItems = append(metricItems, metricItem) + case v1.SegmentMemoryMetricKey: + metricItem := newMetricItem(v1.SegmentMemoryMetricKey, 2, MemoryGroupKey) + metricItem.AddAxi("Segment Memory", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, false) + metricItem.AddLine("Segment Memory", "Segment Memory", + "Memory use of all open segments.", + "group1", "payload.elasticsearch.shard_stats.segments.memory_in_bytes", "max", bucketSizeStr, "", "bytes", "0,0.[00]", "0,0.[00]", false, false) + metricItems = append(metricItems, metricItem) } metrics, err = h.getSingleIndexMetrics(context.Background(), metricItems, query, bucketSize) if err != nil { diff --git a/modules/elastic/api/v1/index_metrics.go b/modules/elastic/api/v1/index_metrics.go index 070b4161..ebbb52e8 100644 --- a/modules/elastic/api/v1/index_metrics.go +++ b/modules/elastic/api/v1/index_metrics.go @@ -73,6 +73,11 @@ const ( SegmentIndexWriterMemoryMetricKey = "segment_index_writer_memory" SegmentTermVectorsMemoryMetricKey = "segment_term_vectors_memory" DocPercentMetricKey = "doc_percent" + SegmentNormsMetricKey = "segment_norms_memory" + SegmentPointsMetricKey = "segment_points_memory" + VersionMapMetricKey = "segment_version_map" + FixedBitSetMetricKey = "segment_fixed_bit_set" + ) func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clusterID string, bucketSize int, min, max int64, indexName string, top int, metricKey string) (map[string]*common.MetricItem, error){ @@ -627,7 +632,54 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu FormatType: "bytes", Units: "", }) - + case SegmentNormsMetricKey: + segmentNormsMetric := newMetricItem(SegmentNormsMetricKey, 17, MemoryGroupKey) + segmentNormsMetric.AddAxi("Segment norms memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: SegmentNormsMetricKey, + Field: "payload.elasticsearch.index_stats.total.segments.norms_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentNormsMetric, + FormatType: "bytes", + Units: "", + }) + case SegmentPointsMetricKey: + segmentPointsMetric := newMetricItem(SegmentPointsMetricKey, 18, MemoryGroupKey) + segmentPointsMetric.AddAxi("Segment points memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: SegmentPointsMetricKey, + Field: "payload.elasticsearch.index_stats.total.segments.points_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentPointsMetric, + FormatType: "bytes", + Units: "", + }) + case VersionMapMetricKey: + segmentVersionMapMetric := newMetricItem(VersionMapMetricKey, 18, MemoryGroupKey) + segmentVersionMapMetric.AddAxi("Segment version map memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: VersionMapMetricKey, + Field: "payload.elasticsearch.index_stats.total.segments.version_map_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentVersionMapMetric, + FormatType: "bytes", + Units: "", + }) + case FixedBitSetMetricKey: + segmentFixedBitSetMetric := newMetricItem(FixedBitSetMetricKey, 18, MemoryGroupKey) + segmentFixedBitSetMetric.AddAxi("Segment fixed bit set memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: FixedBitSetMetricKey, + Field: "payload.elasticsearch.index_stats.total.segments.fixed_bit_set_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentFixedBitSetMetric, + FormatType: "bytes", + Units: "", + }) } diff --git a/modules/elastic/api/v1/index_overview.go b/modules/elastic/api/v1/index_overview.go index c1f019ba..b04a519c 100644 --- a/modules/elastic/api/v1/index_overview.go +++ b/modules/elastic/api/v1/index_overview.go @@ -562,6 +562,13 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ return value / value2 } metricItems = append(metricItems, metricItem) + case SegmentMemoryMetricKey: + metricItem := newMetricItem(SegmentMemoryMetricKey, 2, MemoryGroupKey) + metricItem.AddAxi("Segment Memory", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, false) + metricItem.AddLine("Segment Memory", "Segment Memory", + "Memory use of all open segments.", + "group1", "payload.elasticsearch.index_stats.total.segments.memory_in_bytes", "max", bucketSizeStr, "", "bytes", "0,0.[00]", "0,0.[00]", false, false) + metricItems = append(metricItems, metricItem) } metrics, err = h.getSingleMetrics(ctx, metricItems, query, bucketSize) if err != nil {