feat: splitting metric query, adding query param key for query metric api to support single metric query (#7)

* feat: support query es cluster metrics with special key
* feat: adding context param to control metric request timeout
* feat: splitting metric query, adding query param `key` for query metric  api to support single metric query
* chore: clean unused code
* fix: wrong metric key

modules/elastic/api/cluster_overview.go

@@ -24,6 +24,7 @@
 package api
 
 import (
+	"context"
 	"fmt"
 	"infini.sh/framework/modules/elastic/adapter"
 	"net/http"
@@ -254,7 +255,7 @@ func (h *APIHandler) FetchClusterInfo(w http.ResponseWriter, req *http.Request,
 			},
 		},
 	}
-	indexMetrics := h.getMetrics(query, indexMetricItems, bucketSize)
+	indexMetrics := h.getMetrics(context.Background(), query, indexMetricItems, bucketSize)
 	indexingMetricData := util.MapStr{}
 	for _, line := range indexMetrics["cluster_indexing"].Lines {
 		// remove first metric dot

modules/elastic/api/host.go

@@ -28,6 +28,7 @@
 package api
 
 import (
+	"context"
 	"fmt"
 	log "github.com/cihub/seelog"
 	httprouter "infini.sh/framework/core/api/router"
@@ -604,10 +605,10 @@ func (h *APIHandler) getSingleHostMetric(agentID string, min, max int64, bucketS
 			},
 		},
 	}
-	return h.getSingleMetrics(metricItems, query, bucketSize)
+	return h.getSingleMetrics(context.Background(), metricItems, query, bucketSize)
 }
 
-func (h *APIHandler) getSingleHostMetricFromNode(nodeID string, min, max int64, bucketSize int) map[string]*common.MetricItem {
+func (h *APIHandler) getSingleHostMetricFromNode(ctx context.Context, nodeID string, min, max int64, bucketSize int) map[string]*common.MetricItem {
 	var must = []util.MapStr{
 		{
 			"term": util.MapStr{
@@ -669,7 +670,7 @@ func (h *APIHandler) getSingleHostMetricFromNode(nodeID string, min, max int64,
 		return 100 - value*100/value2
 	}
 	metricItems = append(metricItems, metricItem)
-	return h.getSingleMetrics(metricItems, query, bucketSize)
+	return h.getSingleMetrics(ctx, metricItems, query, bucketSize)
 }
 
 func (h *APIHandler) GetSingleHostMetrics(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
@@ -696,7 +697,7 @@ func (h *APIHandler) GetSingleHostMetrics(w http.ResponseWriter, req *http.Reque
 		return
 	}
 	if hostInfo.AgentID == "" {
-		resBody["metrics"] = h.getSingleHostMetricFromNode(hostInfo.NodeID, min, max, bucketSize)
+		resBody["metrics"] = h.getSingleHostMetricFromNode(context.Background(), hostInfo.NodeID, min, max, bucketSize)
 		h.WriteJSON(w, resBody, http.StatusOK)
 		return
 	}
@@ -866,7 +867,7 @@ func (h *APIHandler) getGroupHostMetric(agentIDs []string, min, max int64, bucke
 			},
 		},
 	}
-	return h.getMetrics(query, hostMetricItems, bucketSize)
+	return h.getMetrics(context.Background(), query, hostMetricItems, bucketSize)
 }
 
 func getHost(hostID string) (*host.HostInfo, error) {

modules/elastic/api/index_metrics.go

@@ -24,8 +24,10 @@
 package api
 
 import (
+	"context"
 	"fmt"
 	log "github.com/cihub/seelog"
+	v1 "infini.sh/console/modules/elastic/api/v1"
 	"infini.sh/framework/core/elastic"
 	"infini.sh/framework/core/global"
 	"infini.sh/framework/core/radix"
@@ -38,7 +40,7 @@ import (
 	"time"
 )
 
-func (h *APIHandler) getIndexMetrics(req *http.Request, clusterID string, bucketSize int, min, max int64, indexName string, top int, shardID string) (map[string]*common.MetricItem, error){
+func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clusterID string, bucketSize int, min, max int64, indexName string, top int, shardID string, metricKey string) (map[string]*common.MetricItem, error){
 	bucketSizeStr:=fmt.Sprintf("%vs",bucketSize)
 	clusterUUID, err := adapter.GetClusterUUID(clusterID)
 	if err != nil {
@@ -141,12 +143,13 @@ func (h *APIHandler) getIndexMetrics(req *http.Request, clusterID string, bucket
 			},
 		},
 	}
-	//索引存储大小
+	indexMetricItems := []GroupMetricItem{}
-	indexStorageMetric := newMetricItem("index_storage", 1, StorageGroupKey)
+	switch metricKey {
-	indexStorageMetric.AddAxi("Index storage","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
+	case v1.IndexStorageMetricKey:
-
+		//索引存储大小
-	indexMetricItems := []GroupMetricItem{
+		indexStorageMetric := newMetricItem(v1.IndexStorageMetricKey, 1, StorageGroupKey)
-		{
+		indexStorageMetric.AddAxi("Index storage","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
 			Key: "index_storage",
 			Field: "payload.elasticsearch.shard_stats.store.size_in_bytes",
 			ID: util.GetUUID(),
@@ -154,451 +157,463 @@ func (h *APIHandler) getIndexMetrics(req *http.Request, clusterID string, bucket
 			MetricItem: indexStorageMetric,
 			FormatType: "bytes",
 			Units: "",
-		},
+		})
+	case v1.SegmentCountMetricKey:
+		// segment 数量
+		segmentCountMetric:=newMetricItem(v1.SegmentCountMetricKey, 15, StorageGroupKey)
+		segmentCountMetric.AddAxi("segment count","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
+		indexMetricItems=append(indexMetricItems, GroupMetricItem{
+			Key: "segment_count",
+			Field: "payload.elasticsearch.shard_stats.segments.count",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: segmentCountMetric,
+			FormatType: "num",
+			Units: "",
+		})
+	case v1.DocCountMetricKey:
+		//索引文档个数
+		docCountMetric := newMetricItem(v1.DocCountMetricKey, 2, DocumentGroupKey)
+		docCountMetric.AddAxi("Doc count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "doc_count",
+			Field: "payload.elasticsearch.shard_stats.docs.count",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: docCountMetric,
+			FormatType: "num",
+			Units: "",
+		})
+	case v1.DocsDeletedMetricKey:
+		// docs 删除数量
+		docsDeletedMetric:=newMetricItem(v1.DocsDeletedMetricKey, 17, DocumentGroupKey)
+		docsDeletedMetric.AddAxi("docs deleted","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
+		indexMetricItems=append(indexMetricItems, GroupMetricItem{
+			Key: "docs_deleted",
+			Field: "payload.elasticsearch.shard_stats.docs.deleted",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: docsDeletedMetric,
+			FormatType: "num",
+			Units: "",
+		})
+	case v1.QueryTimesMetricKey:
+		//查询次数
+		queryTimesMetric := newMetricItem("query_times", 2, OperationGroupKey)
+		queryTimesMetric.AddAxi("Query times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "query_times",
+			Field: "payload.elasticsearch.shard_stats.search.query_total",
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: queryTimesMetric,
+			FormatType: "num",
+			Units: "requests/s",
+		})
+	case v1.FetchTimesMetricKey:
+		//Fetch次数
+		fetchTimesMetric := newMetricItem(v1.FetchTimesMetricKey, 3, OperationGroupKey)
+		fetchTimesMetric.AddAxi("Fetch times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "fetch_times",
+			Field: "payload.elasticsearch.shard_stats.search.fetch_total",
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: fetchTimesMetric,
+			FormatType: "num",
+			Units: "requests/s",
+		})
+	case v1.ScrollTimesMetricKey:
+		//scroll 次数
+		scrollTimesMetric := newMetricItem(v1.ScrollTimesMetricKey, 4, OperationGroupKey)
+		scrollTimesMetric.AddAxi("scroll times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "scroll_times",
+			Field: "payload.elasticsearch.shard_stats.search.scroll_total",
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: scrollTimesMetric,
+			FormatType: "num",
+			Units: "requests/s",
+		})
+	case v1.MergeTimesMetricKey:
+		//Merge次数
+		mergeTimesMetric := newMetricItem(v1.MergeTimesMetricKey, 7, OperationGroupKey)
+		mergeTimesMetric.AddAxi("Merge times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "merge_times",
+			Field: "payload.elasticsearch.shard_stats.merges.total",
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: mergeTimesMetric,
+			FormatType: "num",
+			Units: "requests/s",
+		})
+	case v1.RefreshTimesMetricKey:
+		//Refresh次数
+		refreshTimesMetric := newMetricItem(v1.RefreshTimesMetricKey, 5, OperationGroupKey)
+		refreshTimesMetric.AddAxi("Refresh times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "refresh_times",
+			Field: "payload.elasticsearch.shard_stats.refresh.total",
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: refreshTimesMetric,
+			FormatType: "num",
+			Units: "requests/s",
+		})
+	case v1.FlushTimesMetricKey:
+		//flush 次数
+		flushTimesMetric := newMetricItem(v1.FlushTimesMetricKey, 6, OperationGroupKey)
+		flushTimesMetric.AddAxi("flush times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "flush_times",
+			Field: "payload.elasticsearch.shard_stats.flush.total",
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: flushTimesMetric,
+			FormatType: "num",
+			Units: "requests/s",
+		})
+	case v1.IndexingRateMetricKey:
+		//写入速率
+		indexingRateMetric := newMetricItem(v1.IndexingRateMetricKey, 1, OperationGroupKey)
+		if shardID == "" {
+			indexingRateMetric.OnlyPrimary = true
+		}
+		indexingRateMetric.AddAxi("Indexing rate","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "indexing_rate",
+			Field: "payload.elasticsearch.shard_stats.indexing.index_total",
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: indexingRateMetric,
+			FormatType: "num",
+			Units: "doc/s",
+		})
+	case v1.IndexingBytesMetricKey:
+		indexingBytesMetric := newMetricItem(v1.IndexingBytesMetricKey, 2, OperationGroupKey)
+		if shardID == "" {
+			indexingBytesMetric.OnlyPrimary = true
+		}
+		indexingBytesMetric.AddAxi("Indexing bytes","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key:          "indexing_bytes",
+			Field:        "payload.elasticsearch.shard_stats.store.size_in_bytes",
+			ID:           util.GetUUID(),
+			IsDerivative: true,
+			MetricItem:   indexingBytesMetric,
+			FormatType:   "bytes",
+			Units:        "bytes/s",
+		})
+	case v1.IndexingLatencyMetricKey:
+		//写入时延
+		indexingLatencyMetric := newMetricItem(v1.IndexingLatencyMetricKey, 1, LatencyGroupKey)
+		if shardID == "" {
+			indexingLatencyMetric.OnlyPrimary = true
+		}
+		indexingLatencyMetric.AddAxi("Indexing latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "indexing_latency",
+			Field: "payload.elasticsearch.shard_stats.indexing.index_time_in_millis",
+			Field2: "payload.elasticsearch.shard_stats.indexing.index_total",
+			Calc: func(value, value2 float64) float64 {
+				return value/value2
+			},
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: indexingLatencyMetric,
+			FormatType: "num",
+			Units: "ms",
+		})
+	case v1.QueryLatencyMetricKey:
+		//查询时延
+		queryLatencyMetric := newMetricItem(v1.QueryLatencyMetricKey, 2, LatencyGroupKey)
+		queryLatencyMetric.AddAxi("Query latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "query_latency",
+			Field: "payload.elasticsearch.shard_stats.search.query_time_in_millis",
+			Field2: "payload.elasticsearch.shard_stats.search.query_total",
+			Calc: func(value, value2 float64) float64 {
+				return value/value2
+			},
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: queryLatencyMetric,
+			FormatType: "num",
+			Units: "ms",
+		})
+	case FetchLatencyMetricKey:
+		//fetch时延
+		fetchLatencyMetric := newMetricItem(v1.FetchLatencyMetricKey, 3, LatencyGroupKey)
+		fetchLatencyMetric.AddAxi("Fetch latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "fetch_latency",
+			Field: "payload.elasticsearch.shard_stats.search.fetch_time_in_millis",
+			Field2: "payload.elasticsearch.shard_stats.search.fetch_total",
+			Calc: func(value, value2 float64) float64 {
+				return value/value2
+			},
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: fetchLatencyMetric,
+			FormatType: "num",
+			Units: "ms",
+		})
+	case v1.MergeLatencyMetricKey:
+		//merge时延
+		mergeLatencyMetric := newMetricItem(v1.MergeLatencyMetricKey, 7, LatencyGroupKey)
+		mergeLatencyMetric.AddAxi("Merge latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "merge_latency",
+			Field: "payload.elasticsearch.shard_stats.merges.total_time_in_millis",
+			Field2: "payload.elasticsearch.shard_stats.merges.total",
+			Calc: func(value, value2 float64) float64 {
+				return value/value2
+			},
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: mergeLatencyMetric,
+			FormatType: "num",
+			Units: "ms",
+		})
+	case RefreshLatencyMetricKey:
+		//refresh时延
+		refreshLatencyMetric := newMetricItem(v1.RefreshLatencyMetricKey, 5, LatencyGroupKey)
+		refreshLatencyMetric.AddAxi("Refresh latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "refresh_latency",
+			Field: "payload.elasticsearch.shard_stats.refresh.total_time_in_millis",
+			Field2: "payload.elasticsearch.shard_stats.refresh.total",
+			Calc: func(value, value2 float64) float64 {
+				return value/value2
+			},
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: refreshLatencyMetric,
+			FormatType: "num",
+			Units: "ms",
+		})
+	case v1.ScrollLatencyMetricKey:
+		//scroll时延
+		scrollLatencyMetric := newMetricItem(v1.ScrollLatencyMetricKey, 4, LatencyGroupKey)
+		scrollLatencyMetric.AddAxi("Scroll Latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "scroll_latency",
+			Field: "payload.elasticsearch.shard_stats.search.scroll_time_in_millis",
+			Field2: "payload.elasticsearch.shard_stats.search.scroll_total",
+			Calc: func(value, value2 float64) float64 {
+				return value/value2
+			},
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: scrollLatencyMetric,
+			FormatType: "num",
+			Units: "ms",
+		})
+	case v1.FlushLatencyMetricKey:
+		//flush 时延
+		flushLatencyMetric := newMetricItem(v1.FlushLatencyMetricKey, 6, LatencyGroupKey)
+		flushLatencyMetric.AddAxi("Flush latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "flush_latency",
+			Field: "payload.elasticsearch.shard_stats.flush.total_time_in_millis",
+			Field2: "payload.elasticsearch.shard_stats.flush.total",
+			Calc: func(value, value2 float64) float64 {
+				return value/value2
+			},
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: flushLatencyMetric,
+			FormatType: "num",
+			Units: "ms",
+		})
+	case v1.QueryCacheMetricKey:
+		//queryCache
+		queryCacheMetric := newMetricItem(v1.QueryCacheMetricKey, 1, CacheGroupKey)
+		queryCacheMetric.AddAxi("Query cache","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "query_cache",
+			Field: "payload.elasticsearch.shard_stats.query_cache.memory_size_in_bytes",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: queryCacheMetric,
+			FormatType: "bytes",
+			Units: "",
+		})
+	case v1.RequestCacheMetricKey:
+		//requestCache
+		requestCacheMetric := newMetricItem(v1.RequestCacheMetricKey, 2, CacheGroupKey)
+		requestCacheMetric.AddAxi("request cache","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "request_cache",
+			Field: "payload.elasticsearch.shard_stats.request_cache.memory_size_in_bytes",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: requestCacheMetric,
+			FormatType: "bytes",
+			Units: "",
+		})
+	case v1.RequestCacheHitMetricKey:
+		// Request Cache Hit
+		requestCacheHitMetric:=newMetricItem(v1.RequestCacheHitMetricKey, 6, CacheGroupKey)
+		requestCacheHitMetric.AddAxi("request cache hit","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
+		indexMetricItems=append(indexMetricItems, GroupMetricItem{
+			Key: "request_cache_hit",
+			Field: "payload.elasticsearch.shard_stats.request_cache.hit_count",
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: requestCacheHitMetric,
+			FormatType: "num",
+			Units: "hits",
+		})
+	case  v1.RequestCacheMissMetricKey:
+		// Request Cache Miss
+		requestCacheMissMetric:=newMetricItem(v1.RequestCacheMissMetricKey, 8, CacheGroupKey)
+		requestCacheMissMetric.AddAxi("request cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
+		indexMetricItems=append(indexMetricItems, GroupMetricItem{
+			Key: "request_cache_miss",
+			Field: "payload.elasticsearch.shard_stats.request_cache.miss_count",
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: requestCacheMissMetric,
+			FormatType: "num",
+			Units: "misses",
+		})
+	case v1.QueryCacheCountMetricKey:
+		// Query Cache Count
+		queryCacheCountMetric:=newMetricItem(v1.QueryCacheCountMetricKey, 4, CacheGroupKey)
+		queryCacheCountMetric.AddAxi("query cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
+		indexMetricItems=append(indexMetricItems, GroupMetricItem{
+			Key: "query_cache_count",
+			Field: "payload.elasticsearch.shard_stats.query_cache.cache_count",
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: queryCacheCountMetric,
+			FormatType: "num",
+			Units: "",
+		})
+	case v1.QueryCacheHitMetricKey:
+		// Query Cache Miss
+		queryCacheHitMetric:=newMetricItem(v1.QueryCacheHitMetricKey, 5, CacheGroupKey)
+		queryCacheHitMetric.AddAxi("query cache hit","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
+		indexMetricItems=append(indexMetricItems, GroupMetricItem{
+			Key: "query_cache_hit",
+			Field: "payload.elasticsearch.shard_stats.query_cache.hit_count",
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: queryCacheHitMetric,
+			FormatType: "num",
+			Units: "hits",
+		})
+	case v1.QueryCacheMissMetricKey:
+		// Query Cache Miss
+		queryCacheMissMetric:=newMetricItem(v1.QueryCacheMissMetricKey, 7, CacheGroupKey)
+		queryCacheMissMetric.AddAxi("query cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
+		indexMetricItems=append(indexMetricItems, GroupMetricItem{
+			Key: "query_cache_miss",
+			Field: "payload.elasticsearch.shard_stats.query_cache.miss_count",
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: queryCacheMissMetric,
+			FormatType: "num",
+			Units: "misses",
+		})
+	case v1.FielddataCacheMetricKey:
+		// Fielddata内存占用大小
+		fieldDataCacheMetric:=newMetricItem(v1.FielddataCacheMetricKey, 3, CacheGroupKey)
+		fieldDataCacheMetric.AddAxi("FieldData Cache","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true)
+		indexMetricItems=append(indexMetricItems, GroupMetricItem{
+			Key: "fielddata_cache",
+			Field: "payload.elasticsearch.shard_stats.fielddata.memory_size_in_bytes",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: fieldDataCacheMetric,
+			FormatType: "bytes",
+			Units: "",
+		})
+	case v1.SegmentMemoryMetricKey:
+		//segment memory
+		segmentMemoryMetric := newMetricItem(v1.SegmentMemoryMetricKey, 13, MemoryGroupKey)
+		segmentMemoryMetric.AddAxi("Segment memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "segment_memory",
+			Field: "payload.elasticsearch.shard_stats.segments.memory_in_bytes",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: segmentMemoryMetric,
+			FormatType: "bytes",
+			Units: "",
+		})
+	case v1.SegmentDocValuesMemoryMetricKey:
+		//segment doc values memory
+		docValuesMemoryMetric := newMetricItem(v1.SegmentDocValuesMemoryMetricKey, 13, MemoryGroupKey)
+		docValuesMemoryMetric.AddAxi("Segment Doc values Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "segment_doc_values_memory",
+			Field: "payload.elasticsearch.shard_stats.segments.doc_values_memory_in_bytes",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: docValuesMemoryMetric,
+			FormatType: "bytes",
+			Units: "",
+		})
+	case v1.SegmentTermsMemoryMetricKey:
+		//segment terms memory
+		termsMemoryMetric := newMetricItem(v1.SegmentTermsMemoryMetricKey, 13, MemoryGroupKey)
+		termsMemoryMetric.AddAxi("Segment Terms Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "segment_terms_memory",
+			Field: "payload.elasticsearch.shard_stats.segments.terms_memory_in_bytes",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: termsMemoryMetric,
+			FormatType: "bytes",
+			Units: "",
+		})
+	case v1.SegmentFieldsMemoryMetricKey:
+		//segment fields memory
+		fieldsMemoryMetric := newMetricItem(v1.SegmentFieldsMemoryMetricKey, 13, MemoryGroupKey)
+		fieldsMemoryMetric.AddAxi("Segment Fields Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "segment_fields_memory",
+			Field: "payload.elasticsearch.index_stats.total.segments.stored_fields_memory_in_bytes",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: fieldsMemoryMetric,
+			FormatType: "bytes",
+			Units: "",
+		})
+	case v1.SegmentIndexWriterMemoryMetricKey:
+		// segment index writer memory
+		segmentIndexWriterMemoryMetric:=newMetricItem(v1.SegmentIndexWriterMemoryMetricKey, 16, MemoryGroupKey)
+		segmentIndexWriterMemoryMetric.AddAxi("segment doc values memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true)
+		indexMetricItems=append(indexMetricItems, GroupMetricItem{
+			Key: "segment_index_writer_memory",
+			Field: "payload.elasticsearch.shard_stats.segments.index_writer_memory_in_bytes",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: segmentIndexWriterMemoryMetric,
+			FormatType: "bytes",
+			Units: "",
+		})
+	case v1.SegmentTermVectorsMemoryMetricKey:
+		// segment term vectors memory
+		segmentTermVectorsMemoryMetric:=newMetricItem(v1.SegmentTermVectorsMemoryMetricKey, 16, MemoryGroupKey)
+		segmentTermVectorsMemoryMetric.AddAxi("segment term vectors memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true)
+		indexMetricItems=append(indexMetricItems, GroupMetricItem{
+			Key: "segment_term_vectors_memory",
+			Field: "payload.elasticsearch.shard_stats.segments.term_vectors_memory_in_bytes",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: segmentTermVectorsMemoryMetric,
+			FormatType: "bytes",
+			Units: "",
+		})
 	}
-	// segment 数量
-	segmentCountMetric:=newMetricItem("segment_count", 15, StorageGroupKey)
-	segmentCountMetric.AddAxi("segment count","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
-	indexMetricItems=append(indexMetricItems, GroupMetricItem{
-		Key: "segment_count",
-		Field: "payload.elasticsearch.shard_stats.segments.count",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: segmentCountMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	//索引文档个数
-	docCountMetric := newMetricItem("doc_count", 2, DocumentGroupKey)
-	docCountMetric.AddAxi("Doc count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
 
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "doc_count",
-		Field: "payload.elasticsearch.shard_stats.docs.count",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: docCountMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	// docs 删除数量
-	docsDeletedMetric:=newMetricItem("docs_deleted", 17, DocumentGroupKey)
-	docsDeletedMetric.AddAxi("docs deleted","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
-	indexMetricItems=append(indexMetricItems, GroupMetricItem{
-		Key: "docs_deleted",
-		Field: "payload.elasticsearch.shard_stats.docs.deleted",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: docsDeletedMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	//查询次数
-	queryTimesMetric := newMetricItem("query_times", 2, OperationGroupKey)
-	queryTimesMetric.AddAxi("Query times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "query_times",
-		Field: "payload.elasticsearch.shard_stats.search.query_total",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: queryTimesMetric,
-		FormatType: "num",
-		Units: "requests/s",
-	})
-
-	//Fetch次数
-	fetchTimesMetric := newMetricItem("fetch_times", 3, OperationGroupKey)
-	fetchTimesMetric.AddAxi("Fetch times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "fetch_times",
-		Field: "payload.elasticsearch.shard_stats.search.fetch_total",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: fetchTimesMetric,
-		FormatType: "num",
-		Units: "requests/s",
-	})
-	//scroll 次数
-	scrollTimesMetric := newMetricItem("scroll_times", 4, OperationGroupKey)
-	scrollTimesMetric.AddAxi("scroll times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "scroll_times",
-		Field: "payload.elasticsearch.shard_stats.search.scroll_total",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: scrollTimesMetric,
-		FormatType: "num",
-		Units: "requests/s",
-	})
-	//Merge次数
-	mergeTimesMetric := newMetricItem("merge_times", 7, OperationGroupKey)
-	mergeTimesMetric.AddAxi("Merge times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "merge_times",
-		Field: "payload.elasticsearch.shard_stats.merges.total",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: mergeTimesMetric,
-		FormatType: "num",
-		Units: "requests/s",
-	})
-	//Refresh次数
-	refreshTimesMetric := newMetricItem("refresh_times", 5, OperationGroupKey)
-	refreshTimesMetric.AddAxi("Refresh times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "refresh_times",
-		Field: "payload.elasticsearch.shard_stats.refresh.total",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: refreshTimesMetric,
-		FormatType: "num",
-		Units: "requests/s",
-	})
-	//flush 次数
-	flushTimesMetric := newMetricItem("flush_times", 6, OperationGroupKey)
-	flushTimesMetric.AddAxi("flush times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "flush_times",
-		Field: "payload.elasticsearch.shard_stats.flush.total",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: flushTimesMetric,
-		FormatType: "num",
-		Units: "requests/s",
-	})
-
-	//写入速率
-	indexingRateMetric := newMetricItem("indexing_rate", 1, OperationGroupKey)
-	if shardID == "" {
-		indexingRateMetric.OnlyPrimary = true
-	}
-	indexingRateMetric.AddAxi("Indexing rate","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "indexing_rate",
-		Field: "payload.elasticsearch.shard_stats.indexing.index_total",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: indexingRateMetric,
-		FormatType: "num",
-		Units: "doc/s",
-	})
-	indexingBytesMetric := newMetricItem("indexing_bytes", 2, OperationGroupKey)
-	if shardID == "" {
-		indexingBytesMetric.OnlyPrimary = true
-	}
-	indexingBytesMetric.AddAxi("Indexing bytes","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key:          "indexing_bytes",
-		Field:        "payload.elasticsearch.shard_stats.store.size_in_bytes",
-		ID:           util.GetUUID(),
-		IsDerivative: true,
-		MetricItem:   indexingBytesMetric,
-		FormatType:   "bytes",
-		Units:        "bytes/s",
-	})
-	//写入时延
-	indexingLatencyMetric := newMetricItem("indexing_latency", 1, LatencyGroupKey)
-	if shardID == "" {
-		indexingLatencyMetric.OnlyPrimary = true
-	}
-	indexingLatencyMetric.AddAxi("Indexing latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "indexing_latency",
-		Field: "payload.elasticsearch.shard_stats.indexing.index_time_in_millis",
-		Field2: "payload.elasticsearch.shard_stats.indexing.index_total",
-		Calc: func(value, value2 float64) float64 {
-			return value/value2
-		},
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: indexingLatencyMetric,
-		FormatType: "num",
-		Units: "ms",
-	})
-
-	//查询时延
-	queryLatencyMetric := newMetricItem("query_latency", 2, LatencyGroupKey)
-	queryLatencyMetric.AddAxi("Query latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "query_latency",
-		Field: "payload.elasticsearch.shard_stats.search.query_time_in_millis",
-		Field2: "payload.elasticsearch.shard_stats.search.query_total",
-		Calc: func(value, value2 float64) float64 {
-			return value/value2
-		},
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: queryLatencyMetric,
-		FormatType: "num",
-		Units: "ms",
-	})
-	//fetch时延
-	fetchLatencyMetric := newMetricItem("fetch_latency", 3, LatencyGroupKey)
-	fetchLatencyMetric.AddAxi("Fetch latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "fetch_latency",
-		Field: "payload.elasticsearch.shard_stats.search.fetch_time_in_millis",
-		Field2: "payload.elasticsearch.shard_stats.search.fetch_total",
-		Calc: func(value, value2 float64) float64 {
-			return value/value2
-		},
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: fetchLatencyMetric,
-		FormatType: "num",
-		Units: "ms",
-	})
-
-	//merge时延
-	mergeLatencyMetric := newMetricItem("merge_latency", 7, LatencyGroupKey)
-	mergeLatencyMetric.AddAxi("Merge latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "merge_latency",
-		Field: "payload.elasticsearch.shard_stats.merges.total_time_in_millis",
-		Field2: "payload.elasticsearch.shard_stats.merges.total",
-		Calc: func(value, value2 float64) float64 {
-			return value/value2
-		},
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: mergeLatencyMetric,
-		FormatType: "num",
-		Units: "ms",
-	})
-	//refresh时延
-	refreshLatencyMetric := newMetricItem("refresh_latency", 5, LatencyGroupKey)
-	refreshLatencyMetric.AddAxi("Refresh latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "refresh_latency",
-		Field: "payload.elasticsearch.shard_stats.refresh.total_time_in_millis",
-		Field2: "payload.elasticsearch.shard_stats.refresh.total",
-		Calc: func(value, value2 float64) float64 {
-			return value/value2
-		},
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: refreshLatencyMetric,
-		FormatType: "num",
-		Units: "ms",
-	})
-	//scroll时延
-	scrollLatencyMetric := newMetricItem("scroll_latency", 4, LatencyGroupKey)
-	scrollLatencyMetric.AddAxi("Scroll Latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "scroll_latency",
-		Field: "payload.elasticsearch.shard_stats.search.scroll_time_in_millis",
-		Field2: "payload.elasticsearch.shard_stats.search.scroll_total",
-		Calc: func(value, value2 float64) float64 {
-			return value/value2
-		},
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: scrollLatencyMetric,
-		FormatType: "num",
-		Units: "ms",
-	})
-	//flush 时延
-	flushLatencyMetric := newMetricItem("flush_latency", 6, LatencyGroupKey)
-	flushLatencyMetric.AddAxi("Flush latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "flush_latency",
-		Field: "payload.elasticsearch.shard_stats.flush.total_time_in_millis",
-		Field2: "payload.elasticsearch.shard_stats.flush.total",
-		Calc: func(value, value2 float64) float64 {
-			return value/value2
-		},
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: flushLatencyMetric,
-		FormatType: "num",
-		Units: "ms",
-	})
-	//queryCache
-	queryCacheMetric := newMetricItem("query_cache", 1, CacheGroupKey)
-	queryCacheMetric.AddAxi("Query cache","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "query_cache",
-		Field: "payload.elasticsearch.shard_stats.query_cache.memory_size_in_bytes",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: queryCacheMetric,
-		FormatType: "bytes",
-		Units: "",
-	})
-	//requestCache
-	requestCacheMetric := newMetricItem("request_cache", 2, CacheGroupKey)
-	requestCacheMetric.AddAxi("request cache","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "request_cache",
-		Field: "payload.elasticsearch.shard_stats.request_cache.memory_size_in_bytes",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: requestCacheMetric,
-		FormatType: "bytes",
-		Units: "",
-	})
-	// Request Cache Hit
-	requestCacheHitMetric:=newMetricItem("request_cache_hit", 6, CacheGroupKey)
-	requestCacheHitMetric.AddAxi("request cache hit","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
-	indexMetricItems=append(indexMetricItems, GroupMetricItem{
-		Key: "request_cache_hit",
-		Field: "payload.elasticsearch.shard_stats.request_cache.hit_count",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: requestCacheHitMetric,
-		FormatType: "num",
-		Units: "hits",
-	})
-	// Request Cache Miss
-	requestCacheMissMetric:=newMetricItem("request_cache_miss", 8, CacheGroupKey)
-	requestCacheMissMetric.AddAxi("request cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
-	indexMetricItems=append(indexMetricItems, GroupMetricItem{
-		Key: "request_cache_miss",
-		Field: "payload.elasticsearch.shard_stats.request_cache.miss_count",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: requestCacheMissMetric,
-		FormatType: "num",
-		Units: "misses",
-	})
-	// Query Cache Count
-	queryCacheCountMetric:=newMetricItem("query_cache_count", 4, CacheGroupKey)
-	queryCacheCountMetric.AddAxi("query cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
-	indexMetricItems=append(indexMetricItems, GroupMetricItem{
-		Key: "query_cache_count",
-		Field: "payload.elasticsearch.shard_stats.query_cache.cache_count",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: queryCacheCountMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	// Query Cache Miss
-	queryCacheHitMetric:=newMetricItem("query_cache_hit", 5, CacheGroupKey)
-	queryCacheHitMetric.AddAxi("query cache hit","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
-	indexMetricItems=append(indexMetricItems, GroupMetricItem{
-		Key: "query_cache_hit",
-		Field: "payload.elasticsearch.shard_stats.query_cache.hit_count",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: queryCacheHitMetric,
-		FormatType: "num",
-		Units: "hits",
-	})
-
-	//// Query Cache evictions
-	//queryCacheEvictionsMetric:=newMetricItem("query_cache_evictions", 11, CacheGroupKey)
-	//queryCacheEvictionsMetric.AddAxi("query cache evictions","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
-	//indexMetricItems=append(indexMetricItems, GroupMetricItem{
-	//	Key: "query_cache_evictions",
-	//	Field: "payload.elasticsearch.index_stats.total.query_cache.evictions",
-	//	ID: util.GetUUID(),
-	//	IsDerivative: true,
-	//	MetricItem: queryCacheEvictionsMetric,
-	//	FormatType: "num",
-	//	Units: "evictions",
-	//})
-
-	// Query Cache Miss
-	queryCacheMissMetric:=newMetricItem("query_cache_miss", 7, CacheGroupKey)
-	queryCacheMissMetric.AddAxi("query cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
-	indexMetricItems=append(indexMetricItems, GroupMetricItem{
-		Key: "query_cache_miss",
-		Field: "payload.elasticsearch.shard_stats.query_cache.miss_count",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: queryCacheMissMetric,
-		FormatType: "num",
-		Units: "misses",
-	})
-	// Fielddata内存占用大小
-	fieldDataCacheMetric:=newMetricItem("fielddata_cache", 3, CacheGroupKey)
-	fieldDataCacheMetric.AddAxi("FieldData Cache","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true)
-	indexMetricItems=append(indexMetricItems, GroupMetricItem{
-		Key: "fielddata_cache",
-		Field: "payload.elasticsearch.shard_stats.fielddata.memory_size_in_bytes",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: fieldDataCacheMetric,
-		FormatType: "bytes",
-		Units: "",
-	})
-	//segment memory
-	segmentMemoryMetric := newMetricItem("segment_memory", 13, MemoryGroupKey)
-	segmentMemoryMetric.AddAxi("Segment memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "segment_memory",
-		Field: "payload.elasticsearch.shard_stats.segments.memory_in_bytes",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: segmentMemoryMetric,
-		FormatType: "bytes",
-		Units: "",
-	})
-
-	//segment doc values memory
-	docValuesMemoryMetric := newMetricItem("segment_doc_values_memory", 13, MemoryGroupKey)
-	docValuesMemoryMetric.AddAxi("Segment Doc values Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "segment_doc_values_memory",
-		Field: "payload.elasticsearch.shard_stats.segments.doc_values_memory_in_bytes",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: docValuesMemoryMetric,
-		FormatType: "bytes",
-		Units: "",
-	})
-
-	//segment terms memory
-	termsMemoryMetric := newMetricItem("segment_terms_memory", 13, MemoryGroupKey)
-	termsMemoryMetric.AddAxi("Segment Terms Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "segment_terms_memory",
-		Field: "payload.elasticsearch.shard_stats.segments.terms_memory_in_bytes",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: termsMemoryMetric,
-		FormatType: "bytes",
-		Units: "",
-	})
-
-	//segment fields memory
-	fieldsMemoryMetric := newMetricItem("segment_fields_memory", 13, MemoryGroupKey)
-	fieldsMemoryMetric.AddAxi("Segment Fields Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "segment_fields_memory",
-		Field: "payload.elasticsearch.index_stats.total.segments.stored_fields_memory_in_bytes",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: fieldsMemoryMetric,
-		FormatType: "bytes",
-		Units: "",
-	})
-	// segment index writer memory
-	segmentIndexWriterMemoryMetric:=newMetricItem("segment_index_writer_memory", 16, MemoryGroupKey)
-	segmentIndexWriterMemoryMetric.AddAxi("segment doc values memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true)
-	indexMetricItems=append(indexMetricItems, GroupMetricItem{
-		Key: "segment_index_writer_memory",
-		Field: "payload.elasticsearch.shard_stats.segments.index_writer_memory_in_bytes",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: segmentIndexWriterMemoryMetric,
-		FormatType: "bytes",
-		Units: "",
-	})
-	// segment term vectors memory
-	segmentTermVectorsMemoryMetric:=newMetricItem("segment_term_vectors_memory", 16, MemoryGroupKey)
-	segmentTermVectorsMemoryMetric.AddAxi("segment term vectors memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true)
-	indexMetricItems=append(indexMetricItems, GroupMetricItem{
-		Key: "segment_term_vectors_memory",
-		Field: "payload.elasticsearch.shard_stats.segments.term_vectors_memory_in_bytes",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: segmentTermVectorsMemoryMetric,
-		FormatType: "bytes",
-		Units: "",
-	})
 
 	aggs:=map[string]interface{}{}
 	sumAggs := util.MapStr{}
@@ -727,7 +742,7 @@ func (h *APIHandler) getIndexMetrics(req *http.Request, clusterID string, bucket
 			},
 		},
 	}
-	return h.getMetrics(query, indexMetricItems, bucketSize), nil
+	return h.getMetrics(ctx, query, indexMetricItems, bucketSize), nil
 
 }
 

modules/elastic/api/index_overview.go

@@ -28,6 +28,7 @@
 package api
 
 import (
+	"context"
 	"fmt"
 	log "github.com/cihub/seelog"
 	httprouter "infini.sh/framework/core/api/router"
@@ -40,6 +41,7 @@ import (
 	"infini.sh/framework/modules/elastic/common"
 	"net/http"
 	"strings"
+	"time"
 )
 
 func (h *APIHandler) SearchIndexMetadata(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
@@ -503,7 +505,7 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter,  req *http.Request, p
 			},
 		},
 	}
-	metrics := h.getMetrics(query, nodeMetricItems, bucketSize)
+	metrics := h.getMetrics(context.Background(), query, nodeMetricItems, bucketSize)
 	indexMetrics := map[string]util.MapStr{}
 	for key, item := range metrics {
 		for _, line := range item.Lines {
@@ -851,6 +853,16 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ
 	if bucketSize <= 60 {
 		min = min - int64(2 * bucketSize * 1000)
 	}
+	metricKey := h.GetParameter(req, "key")
+	timeout := h.GetParameterOrDefault(req, "timeout", "60s")
+	du, err := time.ParseDuration(timeout)
+	if err != nil {
+		log.Error(err)
+		h.WriteError(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), du)
+	defer cancel()
 	query := map[string]interface{}{}
 	query["query"] = util.MapStr{
 		"bool": util.MapStr{
@@ -870,76 +882,87 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ
 
 	bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
 	metricItems := []*common.MetricItem{}
-	metricItem:=newMetricItem("index_throughput", 1, OperationGroupKey)
+	metrics := map[string]*common.MetricItem{}
-	metricItem.AddAxi("indexing","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
+	if metricKey == ShardStateMetricKey {
-	if shardID == "" {
+		shardStateMetric, err := h.getIndexShardsMetric(ctx, clusterID, indexName, min, max, bucketSize)
-		metricItem.AddLine("Indexing Rate","Primary Indexing","Number of documents being indexed for node.","group1","payload.elasticsearch.shard_stats.indexing.index_total","max",bucketSizeStr,"doc/s","num","0,0.[00]","0,0.[00]",false,true)
+		if err != nil {
-		metricItem.AddLine("Deleting Rate","Primary Deleting","Number of documents being deleted for node.","group1","payload.elasticsearch.shard_stats.indexing.delete_total","max",bucketSizeStr,"doc/s","num","0,0.[00]","0,0.[00]",false,true)
+			log.Error(err)
-		metricItem.Lines[0].Metric.OnlyPrimary = true
+			h.WriteError(w, err.Error(), http.StatusInternalServerError)
-		metricItem.Lines[1].Metric.OnlyPrimary = true
+			return
-	}else{
+		}
-		metricItem.AddLine("Indexing Rate","Indexing Rate","Number of documents being indexed for node.","group1","payload.elasticsearch.shard_stats.indexing.index_total","max",bucketSizeStr,"doc/s","num","0,0.[00]","0,0.[00]",false,true)
+		metrics["shard_state"] = shardStateMetric
-		metricItem.AddLine("Deleting Rate","Deleting Rate","Number of documents being deleted for node.","group1","payload.elasticsearch.shard_stats.indexing.delete_total","max",bucketSizeStr,"doc/s","num","0,0.[00]","0,0.[00]",false,true)
+	}else {
-	}
+		switch metricKey {
-	metricItems=append(metricItems,metricItem)
+		case IndexThroughputMetricKey:
-	metricItem=newMetricItem("search_throughput", 2, OperationGroupKey)
+			metricItem := newMetricItem("index_throughput", 1, OperationGroupKey)
-	metricItem.AddAxi("searching","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,false)
+			metricItem.AddAxi("indexing", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
-	metricItem.AddLine("Search Rate","Search Rate",
+			if shardID == "" {
-		"Number of search requests being executed.",
+				metricItem.AddLine("Indexing Rate", "Primary Indexing", "Number of documents being indexed for node.", "group1", "payload.elasticsearch.shard_stats.indexing.index_total", "max", bucketSizeStr, "doc/s", "num", "0,0.[00]", "0,0.[00]", false, true)
-		"group1","payload.elasticsearch.shard_stats.search.query_total","max",bucketSizeStr,"query/s","num","0,0.[00]","0,0.[00]",false,true)
+				metricItem.AddLine("Deleting Rate", "Primary Deleting", "Number of documents being deleted for node.", "group1", "payload.elasticsearch.shard_stats.indexing.delete_total", "max", bucketSizeStr, "doc/s", "num", "0,0.[00]", "0,0.[00]", false, true)
-	metricItems=append(metricItems,metricItem)
+				metricItem.Lines[0].Metric.OnlyPrimary = true
+				metricItem.Lines[1].Metric.OnlyPrimary = true
+			} else {
+				metricItem.AddLine("Indexing Rate", "Indexing Rate", "Number of documents being indexed for node.", "group1", "payload.elasticsearch.shard_stats.indexing.index_total", "max", bucketSizeStr, "doc/s", "num", "0,0.[00]", "0,0.[00]", false, true)
+				metricItem.AddLine("Deleting Rate", "Deleting Rate", "Number of documents being deleted for node.", "group1", "payload.elasticsearch.shard_stats.indexing.delete_total", "max", bucketSizeStr, "doc/s", "num", "0,0.[00]", "0,0.[00]", false, true)
+			}
+			metricItems = append(metricItems, metricItem)
+		case SearchThroughputMetricKey:
+			metricItem := newMetricItem("search_throughput", 2, OperationGroupKey)
+			metricItem.AddAxi("searching", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
+			metricItem.AddLine("Search Rate", "Search Rate",
+				"Number of search requests being executed.",
+				"group1", "payload.elasticsearch.shard_stats.search.query_total", "max", bucketSizeStr, "query/s", "num", "0,0.[00]", "0,0.[00]", false, true)
+			metricItems = append(metricItems, metricItem)
+		case IndexLatencyMetricKey:
+			metricItem := newMetricItem("index_latency", 3, LatencyGroupKey)
+			metricItem.AddAxi("indexing", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
+			if shardID == "" { //index level
+				metricItem.AddLine("Indexing Latency", "Primary Indexing Latency", "Average latency for indexing documents.", "group1", "payload.elasticsearch.shard_stats.indexing.index_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
+				metricItem.AddLine("Deleting Latency", "Primary Deleting Latency", "Average latency for delete documents.", "group1", "payload.elasticsearch.shard_stats.indexing.delete_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
+				metricItem.Lines[0].Metric.OnlyPrimary = true
+				metricItem.Lines[1].Metric.OnlyPrimary = true
+			} else { // shard level
+				metricItem.AddLine("Indexing Latency", "Indexing Latency", "Average latency for indexing documents.", "group1", "payload.elasticsearch.shard_stats.indexing.index_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
+				metricItem.AddLine("Deleting Latency", "Deleting Latency", "Average latency for delete documents.", "group1", "payload.elasticsearch.shard_stats.indexing.delete_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
+			}
+			metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.shard_stats.indexing.index_total"
+			metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
+				return value / value2
+			}
+			metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.shard_stats.indexing.delete_total"
+			metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
+				return value / value2
+			}
+			metricItems = append(metricItems, metricItem)
+		case SearchLatencyMetricKey:
+			metricItem := newMetricItem("search_latency", 4, LatencyGroupKey)
+			metricItem.AddAxi("searching", "group2", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
 
-	metricItem=newMetricItem("index_latency", 3, LatencyGroupKey)
+			metricItem.AddLine("Searching", "Query Latency", "Average latency for searching query.", "group2", "payload.elasticsearch.shard_stats.search.query_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
-	metricItem.AddAxi("indexing","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
+			metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.shard_stats.search.query_total"
-	if shardID == "" { //index level
+			metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
-		metricItem.AddLine("Indexing Latency","Primary Indexing Latency","Average latency for indexing documents.","group1","payload.elasticsearch.shard_stats.indexing.index_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true)
+				return value / value2
-		metricItem.AddLine("Deleting Latency","Primary Deleting Latency","Average latency for delete documents.","group1","payload.elasticsearch.shard_stats.indexing.delete_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true)
+			}
-		metricItem.Lines[0].Metric.OnlyPrimary = true
+			metricItem.AddLine("Searching", "Fetch Latency", "Average latency for searching fetch.", "group2", "payload.elasticsearch.shard_stats.search.fetch_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
-		metricItem.Lines[1].Metric.OnlyPrimary = true
+			metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.shard_stats.search.fetch_total"
-	}else{ // shard level
+			metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
-		metricItem.AddLine("Indexing Latency","Indexing Latency","Average latency for indexing documents.","group1","payload.elasticsearch.shard_stats.indexing.index_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true)
+				return value / value2
-		metricItem.AddLine("Deleting Latency","Deleting Latency","Average latency for delete documents.","group1","payload.elasticsearch.shard_stats.indexing.delete_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true)
+			}
+			metricItem.AddLine("Searching", "Scroll Latency", "Average latency for searching fetch.", "group2", "payload.elasticsearch.shard_stats.search.scroll_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
+			metricItem.Lines[2].Metric.Field2 = "payload.elasticsearch.shard_stats.search.scroll_total"
+			metricItem.Lines[2].Metric.Calc = func(value, value2 float64) float64 {
+				return value / value2
+			}
+			metricItems = append(metricItems, metricItem)
+		}
+		metrics = h.getSingleIndexMetrics(context.Background(), metricItems, query, bucketSize)
 	}
-	metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.shard_stats.indexing.index_total"
-	metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
-		return value/value2
-	}
-	metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.shard_stats.indexing.delete_total"
-	metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
-		return value/value2
-	}
-	metricItems=append(metricItems,metricItem)
 
-	metricItem=newMetricItem("search_latency", 4, LatencyGroupKey)
-	metricItem.AddAxi("searching","group2",common.PositionLeft,"num","0,0","0,0.[00]",5,false)
-
-	metricItem.AddLine("Searching","Query Latency","Average latency for searching query.","group2","payload.elasticsearch.shard_stats.search.query_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true)
-	metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.shard_stats.search.query_total"
-	metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
-		return value/value2
-	}
-	metricItem.AddLine("Searching","Fetch Latency","Average latency for searching fetch.","group2","payload.elasticsearch.shard_stats.search.fetch_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true)
-	metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.shard_stats.search.fetch_total"
-	metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
-		return value/value2
-	}
-	metricItem.AddLine("Searching","Scroll Latency","Average latency for searching fetch.","group2","payload.elasticsearch.shard_stats.search.scroll_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true)
-	metricItem.Lines[2].Metric.Field2 = "payload.elasticsearch.shard_stats.search.scroll_total"
-	metricItem.Lines[2].Metric.Calc = func(value, value2 float64) float64 {
-		return value/value2
-	}
-	metricItems=append(metricItems,metricItem)
-	metrics := h.getSingleIndexMetrics(metricItems,query, bucketSize)
-	shardStateMetric, err := h.getIndexShardsMetric(clusterID, indexName, min, max, bucketSize)
-	if err != nil {
-		log.Error(err)
-	}
-	metrics["shard_state"] = shardStateMetric
 	resBody["metrics"] = metrics
 	h.WriteJSON(w, resBody, http.StatusOK)
 }
 
-func (h *APIHandler) getIndexShardsMetric(id, indexName string, min, max int64, bucketSize int)(*common.MetricItem, error){
+func (h *APIHandler) getIndexShardsMetric(ctx context.Context, id, indexName string, min, max int64, bucketSize int)(*common.MetricItem, error){
 	bucketSizeStr:=fmt.Sprintf("%vs",bucketSize)
 	intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr)
 	if err != nil {
@@ -1007,7 +1030,8 @@ func (h *APIHandler) getIndexShardsMetric(id, indexName string, min, max int64,
 			},
 		},
 	}
-	response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query))
+	queryDSL := util.MustToJSONBytes(query)
+	response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).QueryDSL(ctx, getAllMetricsIndex(), nil, queryDSL)
 	if err != nil {
 		log.Error(err)
 		return nil, err
@@ -1025,6 +1049,7 @@ func (h *APIHandler) getIndexShardsMetric(id, indexName string, min, max int64,
 	}
 	metricItem.Lines[0].Data = metricData
 	metricItem.Lines[0].Type = common.GraphTypeBar
+	metricItem.Request = string(queryDSL)
 	return metricItem, nil
 }
 

modules/elastic/api/manage.go

@@ -532,6 +532,7 @@ func (h *APIHandler) HandleClusterMetricsAction(w http.ResponseWriter, req *http
 		h.APIHandler.HandleClusterMetricsAction(w, req, ps)
 		return
 	}
+	key := h.GetParameter(req, "key")
 
 	bucketSize, min, max, err := h.getMetricRangeAndBucketSize(req, 10, 90)
 	if err != nil {
@@ -546,18 +547,23 @@ func (h *APIHandler) HandleClusterMetricsAction(w http.ResponseWriter, req *http
 		}
 	}
 
-	//fmt.Println(min," vs ",max,",",rangeFrom,rangeTo,"range hours:",hours)
-
-	//metrics:=h.GetClusterMetrics(id,bucketSize,min,max)
-	isOverview := h.GetIntOrDefault(req, "overview", 0)
 	var metrics interface{}
 	if bucketSize <= 60 {
 		min = min - int64(2*bucketSize*1000)
 	}
-	if isOverview == 1 {
+	timeout := h.GetParameterOrDefault(req, "timeout", "60s")
-		metrics = h.GetClusterIndexMetrics(id, bucketSize, min, max)
+	du, err := time.ParseDuration(timeout)
-	} else {
+	if err != nil {
-		metrics = h.GetClusterMetrics(id, bucketSize, min, max)
+		log.Error(err)
+		h.WriteError(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), du)
+	defer cancel()
+	if util.StringInArray([]string{IndexThroughputMetricKey, SearchThroughputMetricKey, IndexLatencyMetricKey, SearchLatencyMetricKey}, key) {
+		metrics = h.GetClusterIndexMetrics(ctx, id, bucketSize, min, max, key)
+	}else{
+		metrics = h.GetClusterMetrics(ctx, id, bucketSize, min, max, key)
 	}
 
 	resBody["metrics"] = metrics
@@ -584,7 +590,17 @@ func (h *APIHandler) HandleNodeMetricsAction(w http.ResponseWriter, req *http.Re
 	if bucketSize <= 60 {
 		min = min - int64(2*bucketSize*1000)
 	}
-	resBody["metrics"], err = h.getNodeMetrics(id, bucketSize, min, max, nodeName, top)
+	key := h.GetParameter(req, "key")
+	timeout := h.GetParameterOrDefault(req, "timeout", "60s")
+	du, err := time.ParseDuration(timeout)
+	if err != nil {
+		log.Error(err)
+		h.WriteError(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), du)
+	defer cancel()
+	resBody["metrics"], err = h.getNodeMetrics(ctx, id, bucketSize, min, max, nodeName, top, key)
 	if err != nil {
 		log.Error(err)
 		h.WriteError(w, err.Error(), http.StatusInternalServerError)
@@ -627,57 +643,87 @@ func (h *APIHandler) HandleIndexMetricsAction(w http.ResponseWriter, req *http.R
 	if bucketSize <= 60 {
 		min = min - int64(2*bucketSize*1000)
 	}
-	metrics, err := h.getIndexMetrics(req, id, bucketSize, min, max, indexName, top, shardID)
+	key := h.GetParameter(req, "key")
+	timeout := h.GetParameterOrDefault(req, "timeout", "60s")
+	du, err := time.ParseDuration(timeout)
 	if err != nil {
 		log.Error(err)
 		h.WriteError(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
-
+	ctx, cancel := context.WithTimeout(context.Background(), du)
-	if metrics["doc_count"] != nil && metrics["docs_deleted"] != nil && len(metrics["doc_count"].Lines) > 0 && len(metrics["docs_deleted"].Lines) > 0 {
+	defer cancel()
-		metricA := metrics["doc_count"]
+	var metrics map[string]*common.MetricItem
-		metricB := metrics["docs_deleted"]
+	if key == v1.DocPercentMetricKey {
-		if dataA, ok := metricA.Lines[0].Data.([][]interface{}); ok {
+		metrics, err = h.getIndexMetrics(ctx, req, id, bucketSize, min, max, indexName, top, shardID, v1.DocCountMetricKey)
-			if dataB, ok := metricB.Lines[0].Data.([][]interface{}); ok {
+		if err != nil {
-				data := make([]map[string]interface{}, 0, len(dataA)*2)
+			log.Error(err)
-				var (
+			h.WriteError(w, err.Error(), http.StatusInternalServerError)
-					x1 float64
+			return
-					x2 float64
+		}
-				)
+		docsDeletedMetrics, err := h.getIndexMetrics(ctx, req, id, bucketSize, min, max, indexName, top, shardID, v1.DocsDeletedMetricKey)
-				for i := 0; i < len(dataA); i++ {
+		if err != nil {
-					x1 = dataA[i][1].(float64)
+			log.Error(err)
-					x2 = dataB[i][1].(float64)
+			h.WriteError(w, err.Error(), http.StatusInternalServerError)
-					if x1+x2 == 0 {
+			return
-						continue
+		}
-					}
+		for k, v := range docsDeletedMetrics {
-					data = append(data, map[string]interface{}{
+			if v != nil {
-						"x": dataA[i][0],
+				metrics[k] = v
-						"y": x1 / (x1 + x2) * 100,
-						"g": "Doc Count",
-					})
-					data = append(data, map[string]interface{}{
-						"x": dataA[i][0],
-						"y": x2 / (x1 + x2) * 100,
-						"g": "Doc Deleted",
-					})
-				}
-				metricDocPercent := &common.MetricItem{
-					Axis:  []*common.MetricAxis{},
-					Key:   "doc_percent",
-					Group: metricA.Group,
-					Order: 18,
-					Lines: []*common.MetricLine{
-						{
-							TimeRange: metricA.Lines[0].TimeRange,
-							Data:      data,
-							Type:      common.GraphTypeBar,
-						},
-					},
-				}
-				metrics["doc_percent"] = metricDocPercent
 			}
 		}
+		if metrics["doc_count"] != nil && metrics["docs_deleted"] != nil && len(metrics["doc_count"].Lines) > 0 && len(metrics["docs_deleted"].Lines) > 0 {
+			metricA := metrics["doc_count"]
+			metricB := metrics["docs_deleted"]
+			if dataA, ok := metricA.Lines[0].Data.([][]interface{}); ok {
+				if dataB, ok := metricB.Lines[0].Data.([][]interface{}); ok {
+					data := make([]map[string]interface{}, 0, len(dataA)*2)
+					var (
+						x1 float64
+						x2 float64
+					)
+					for i := 0; i < len(dataA); i++ {
+						x1 = dataA[i][1].(float64)
+						x2 = dataB[i][1].(float64)
+						if x1+x2 == 0 {
+							continue
+						}
+						data = append(data, map[string]interface{}{
+							"x": dataA[i][0],
+							"y": x1 / (x1 + x2) * 100,
+							"g": "Doc Count",
+						})
+						data = append(data, map[string]interface{}{
+							"x": dataA[i][0],
+							"y": x2 / (x1 + x2) * 100,
+							"g": "Doc Deleted",
+						})
+					}
+					metricDocPercent := &common.MetricItem{
+						Axis:  []*common.MetricAxis{},
+						Key:   "doc_percent",
+						Group: metricA.Group,
+						Order: 18,
+						Lines: []*common.MetricLine{
+							{
+								TimeRange: metricA.Lines[0].TimeRange,
+								Data:      data,
+								Type:      common.GraphTypeBar,
+							},
+						},
+					}
+					metrics["doc_percent"] = metricDocPercent
+				}
+			}
 
+		}
+	}else{
+		metrics, err = h.getIndexMetrics(ctx, req, id, bucketSize, min, max, indexName, top, shardID, key)
+		if err != nil {
+			log.Error(err)
+			h.WriteError(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
 	}
 	resBody["metrics"] = metrics
 	ver := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).GetVersion()
@@ -711,7 +757,17 @@ func (h *APIHandler) HandleQueueMetricsAction(w http.ResponseWriter, req *http.R
 	if bucketSize <= 60 {
 		min = min - int64(2*bucketSize*1000)
 	}
-	resBody["metrics"], err = h.getThreadPoolMetrics(id, bucketSize, min, max, nodeName, top)
+	key := h.GetParameter(req, "key")
+	timeout := h.GetParameterOrDefault(req, "timeout", "60s")
+	du, err := time.ParseDuration(timeout)
+	if err != nil {
+		log.Error(err)
+		h.WriteError(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), du)
+	defer cancel()
+	resBody["metrics"], err = h.getThreadPoolMetrics(ctx, id, bucketSize, min, max, nodeName, top, key)
 	if err != nil {
 		log.Error(err)
 		h.WriteError(w, err.Error(), http.StatusInternalServerError)
@@ -837,56 +893,96 @@ const (
 	CircuitBreakerGroupKey = "circuit_breaker"
 )
 
-func (h *APIHandler) GetClusterMetrics(id string, bucketSize int, min, max int64) map[string]*common.MetricItem {
+const (
+	ClusterStorageMetricKey = "cluster_storage"
+	ClusterDocumentsMetricKey = "cluster_documents"
+	ClusterIndicesMetricKey  = "cluster_indices"
+	ClusterNodeCountMetricKey = "node_count"
+	ClusterHealthMetricKey = "cluster_health"
+	ShardCountMetricKey = "shard_count"
+	CircuitBreakerMetricKey = "circuit_breaker"
+)
 
+func (h *APIHandler) GetClusterMetrics(ctx context.Context, id string, bucketSize int, min, max int64, metricKey string) map[string]*common.MetricItem {
+
+	var clusterMetricsResult = map[string]*common.MetricItem {}
+	switch metricKey {
+	case ClusterDocumentsMetricKey,
+		ClusterStorageMetricKey,
+		ClusterIndicesMetricKey,
+		ClusterNodeCountMetricKey:
+		clusterMetricsResult = h.getClusterMetricsByKey(ctx, id, bucketSize, min, max, metricKey)
+	case IndexLatencyMetricKey, IndexThroughputMetricKey, SearchThroughputMetricKey, SearchLatencyMetricKey:
+		clusterMetricsResult = h.GetClusterIndexMetrics(ctx, id, bucketSize, min, max, metricKey)
+	case ClusterHealthMetricKey:
+		statusMetric, err := h.getClusterStatusMetric(ctx, id, min, max, bucketSize)
+		if err == nil {
+			clusterMetricsResult[ClusterHealthMetricKey] = statusMetric
+		} else {
+			log.Error("get cluster status metric error: ", err)
+		}
+	case ShardCountMetricKey:
+		clusterMetricsResult = h.getShardsMetric(ctx, id, min, max, bucketSize)
+
+	case CircuitBreakerMetricKey:
+		clusterMetricsResult = h.getCircuitBreakerMetric(ctx, id, min, max, bucketSize)
+	}
+
+	return clusterMetricsResult
+}
+
+func (h *APIHandler) getClusterMetricsByKey(ctx context.Context, id string, bucketSize int, min, max int64, metricKey string) map[string]*common.MetricItem {
 	bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
 
 	clusterMetricItems := []*common.MetricItem{}
-	metricItem := newMetricItem("cluster_storage", 8, StorageGroupKey)
+	switch metricKey {
-	metricItem.AddAxi("indices_storage", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
+	case ClusterStorageMetricKey:
-	metricItem.AddAxi("available_storage", "group2", common.PositionRight, "bytes", "0.[0]", "0.[0]", 5, true)
+		metricItem := newMetricItem("cluster_storage", 8, StorageGroupKey)
+		metricItem.AddAxi("indices_storage", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
+		metricItem.AddAxi("available_storage", "group2", common.PositionRight, "bytes", "0.[0]", "0.[0]", 5, true)
 
-	metricItem.AddLine("Disk", "Indices Storage", "", "group1", "payload.elasticsearch.cluster_stats.indices.store.size_in_bytes", "max", bucketSizeStr, "", "bytes", "0,0.[00]", "0,0.[00]", false, false)
+		metricItem.AddLine("Disk", "Indices Storage", "", "group1", "payload.elasticsearch.cluster_stats.indices.store.size_in_bytes", "max", bucketSizeStr, "", "bytes", "0,0.[00]", "0,0.[00]", false, false)
-	metricItem.AddLine("Disk", "Available Disk", "", "group2", "payload.elasticsearch.cluster_stats.nodes.fs.available_in_bytes", "max", bucketSizeStr, "", "bytes", "0,0.[00]", "0,0.[00]", false, false)
+		metricItem.AddLine("Disk", "Available Disk", "", "group2", "payload.elasticsearch.cluster_stats.nodes.fs.available_in_bytes", "max", bucketSizeStr, "", "bytes", "0,0.[00]", "0,0.[00]", false, false)
 
-	clusterMetricItems = append(clusterMetricItems, metricItem)
+		clusterMetricItems = append(clusterMetricItems, metricItem)
 
-	metricItem = newMetricItem("cluster_documents", 4, StorageGroupKey)
+	case ClusterDocumentsMetricKey:
-	metricItem.AddAxi("count", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
+		metricItem := newMetricItem("cluster_documents", 4, StorageGroupKey)
-	metricItem.AddAxi("deleted", "group2", common.PositionRight, "num", "0,0", "0,0.[00]", 5, false)
+		metricItem.AddAxi("count", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
-	metricItem.AddLine("Documents Count", "Documents Count", "", "group1", "payload.elasticsearch.cluster_stats.indices.docs.count", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false)
+		metricItem.AddAxi("deleted", "group2", common.PositionRight, "num", "0,0", "0,0.[00]", 5, false)
-	metricItem.AddLine("Documents Deleted", "Documents Deleted", "", "group2", "payload.elasticsearch.cluster_stats.indices.docs.deleted", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false)
+		metricItem.AddLine("Documents Count", "Documents Count", "", "group1", "payload.elasticsearch.cluster_stats.indices.docs.count", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false)
-	clusterMetricItems = append(clusterMetricItems, metricItem)
+		metricItem.AddLine("Documents Deleted", "Documents Deleted", "", "group2", "payload.elasticsearch.cluster_stats.indices.docs.deleted", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false)
+		clusterMetricItems = append(clusterMetricItems, metricItem)
+	case ClusterIndicesMetricKey:
+		metricItem := newMetricItem("cluster_indices", 6, StorageGroupKey)
+		metricItem.AddAxi("count", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
+		metricItem.AddLine("Indices Count", "Indices Count", "", "group1", "payload.elasticsearch.cluster_stats.indices.count", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false)
+		clusterMetricItems = append(clusterMetricItems, metricItem)
+	case ClusterNodeCountMetricKey:
+		metricItem := newMetricItem("node_count", 5, MemoryGroupKey)
+		metricItem.AddAxi("count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
+		meta := elastic.GetMetadata(id)
+		if meta == nil {
+			err := fmt.Errorf("metadata of cluster [%s] is not found", id)
+			panic(err)
+		}
+		majorVersion := meta.GetMajorVersion()
 
-	metricItem = newMetricItem("cluster_indices", 6, StorageGroupKey)
+		metricItem.AddLine("Total", "Total Nodes", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.total", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
-	metricItem.AddAxi("count", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
+		if majorVersion < 5 {
-	metricItem.AddLine("Indices Count", "Indices Count", "", "group1", "payload.elasticsearch.cluster_stats.indices.count", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false)
+			metricItem.AddLine("Master Only", "Master Only", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.master_only", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
-	clusterMetricItems = append(clusterMetricItems, metricItem)
+			metricItem.AddLine("Data Node", "Data Only", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.data_only", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
+			metricItem.AddLine("Master Data", "Master Data", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.master_data", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
+		} else {
+			metricItem.AddLine("Master Node", "Master Node", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.master", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
+			metricItem.AddLine("Data Node", "Data Node", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.data", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
+			metricItem.AddLine("Coordinating Node Only", "Coordinating Node Only", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.coordinating_only", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
+			metricItem.AddLine("Ingest Node", "Ingest Node", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.ingest", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
+		}
 
-	metricItem = newMetricItem("node_count", 5, MemoryGroupKey)
+		clusterMetricItems = append(clusterMetricItems, metricItem)
-	metricItem.AddAxi("count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
-	meta := elastic.GetMetadata(id)
-	if meta == nil {
-		err := fmt.Errorf("metadata of cluster [%s] is not found", id)
-		panic(err)
-	}
-	majorVersion := meta.GetMajorVersion()
-
-	metricItem.AddLine("Total", "Total Nodes", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.total", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
-
-	//TODO check version difference
-	if majorVersion < 5 {
-		metricItem.AddLine("Master Only", "Master Only", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.master_only", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
-		metricItem.AddLine("Data Node", "Data Only", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.data_only", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
-		metricItem.AddLine("Master Data", "Master Data", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.master_data", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
-	} else {
-		metricItem.AddLine("Master Node", "Master Node", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.master", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
-		metricItem.AddLine("Data Node", "Data Node", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.data", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
-		metricItem.AddLine("Coordinating Node Only", "Coordinating Node Only", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.coordinating_only", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
-		metricItem.AddLine("Ingest Node", "Ingest Node", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.ingest", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
 	}
 
-	clusterMetricItems = append(clusterMetricItems, metricItem)
 	query := map[string]interface{}{}
 	query["query"] = util.MapStr{
 		"bool": util.MapStr{
@@ -925,79 +1021,70 @@ func (h *APIHandler) GetClusterMetrics(id string, bucketSize int, min, max int64
 			},
 		},
 	}
-	//todo: since there is four queries, we can change these query to async
+	return h.getSingleMetrics(ctx, clusterMetricItems, query, bucketSize)
-	indexMetricsResult := h.GetClusterIndexMetrics(id, bucketSize, min, max)
-	clusterMetricsResult := h.getSingleMetrics(clusterMetricItems, query, bucketSize)
-	for k, v := range clusterMetricsResult {
-		indexMetricsResult[k] = v
-	}
-	statusMetric, err := h.getClusterStatusMetric(id, min, max, bucketSize)
-	if err == nil {
-		indexMetricsResult["cluster_health"] = statusMetric
-	} else {
-		log.Error("get cluster status metric error: ", err)
-	}
-	clusterHealthMetricsResult := h.getShardsMetric(id, min, max, bucketSize)
-	for k, v := range clusterHealthMetricsResult {
-		indexMetricsResult[k] = v
-	}
-	// get CircuitBreaker metric
-	circuitBreakerMetricsResult := h.getCircuitBreakerMetric(id, min, max, bucketSize)
-	for k, v := range circuitBreakerMetricsResult {
-		indexMetricsResult[k] = v
-	}
-
-	return indexMetricsResult
 }
-func (h *APIHandler) GetClusterIndexMetrics(id string, bucketSize int, min, max int64) map[string]*common.MetricItem {
+
+const (
+	IndexThroughputMetricKey = "index_throughput"
+	SearchThroughputMetricKey = "search_throughput"
+	IndexLatencyMetricKey    = "index_latency"
+	SearchLatencyMetricKey   = "search_latency"
+)
+
+func (h *APIHandler) GetClusterIndexMetrics(ctx context.Context, id string, bucketSize int, min, max int64, metricKey string) map[string]*common.MetricItem {
 	bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
 	metricItems := []*common.MetricItem{}
-	metricItem := newMetricItem("index_throughput", 2, OperationGroupKey)
+	switch metricKey {
-	metricItem.AddAxi("indexing", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
+	case IndexThroughputMetricKey:
-	metricItem.AddLine("Indexing Rate", "Total Indexing", "Number of documents being indexed for primary and replica shards.", "group1", "payload.elasticsearch.node_stats.indices.indexing.index_total", "max", bucketSizeStr, "doc/s", "num", "0,0.[00]", "0,0.[00]", false, true)
+		metricItem := newMetricItem(IndexThroughputMetricKey, 2, OperationGroupKey)
-	metricItems = append(metricItems, metricItem)
+		metricItem.AddAxi("indexing", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
+		metricItem.AddLine("Indexing Rate", "Total Indexing", "Number of documents being indexed for primary and replica shards.", "group1", "payload.elasticsearch.node_stats.indices.indexing.index_total", "max", bucketSizeStr, "doc/s", "num", "0,0.[00]", "0,0.[00]", false, true)
+		metricItems = append(metricItems, metricItem)
+	case SearchThroughputMetricKey:
+		metricItem := newMetricItem(SearchThroughputMetricKey, 2, OperationGroupKey)
+		metricItem.AddAxi("searching", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
+		metricItem.AddLine("Search Rate", "Total Query",
+			"Number of search requests being executed across primary and replica shards. A single search can run against multiple shards!",
+			"group1", "payload.elasticsearch.node_stats.indices.search.query_total", "max", bucketSizeStr, "query/s", "num", "0,0.[00]", "0,0.[00]", false, true)
+		metricItems = append(metricItems, metricItem)
+	case IndexLatencyMetricKey:
+		metricItem := newMetricItem(IndexLatencyMetricKey, 3, LatencyGroupKey)
+		metricItem.AddAxi("indexing", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
 
-	metricItem = newMetricItem("search_throughput", 2, OperationGroupKey)
+		metricItem.AddLine("Indexing", "Indexing Latency", "Average latency for indexing documents.", "group1", "payload.elasticsearch.node_stats.indices.indexing.index_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
-	metricItem.AddAxi("searching", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
+		metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.node_stats.indices.indexing.index_total"
-	metricItem.AddLine("Search Rate", "Total Query",
+		metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
-		"Number of search requests being executed across primary and replica shards. A single search can run against multiple shards!",
+			return value / value2
-		"group1", "payload.elasticsearch.node_stats.indices.search.query_total", "max", bucketSizeStr, "query/s", "num", "0,0.[00]", "0,0.[00]", false, true)
+		}
-	metricItems = append(metricItems, metricItem)
+		metricItem.AddLine("Indexing", "Delete Latency", "Average latency for delete documents.", "group1", "payload.elasticsearch.node_stats.indices.indexing.delete_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
+		metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.node_stats.indices.indexing.delete_total"
+		metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
+			return value / value2
+		}
+		metricItems = append(metricItems, metricItem)
+	case SearchLatencyMetricKey:
+		metricItem := newMetricItem(SearchLatencyMetricKey, 3, LatencyGroupKey)
+		metricItem.AddAxi("searching", "group2", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
 
-	metricItem = newMetricItem("index_latency", 3, LatencyGroupKey)
+		metricItem.AddLine("Searching", "Query Latency", "Average latency for searching query.", "group2", "payload.elasticsearch.node_stats.indices.search.query_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
-	metricItem.AddAxi("indexing", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
+		metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.query_total"
-
+		metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
-	metricItem.AddLine("Indexing", "Indexing Latency", "Average latency for indexing documents.", "group1", "payload.elasticsearch.node_stats.indices.indexing.index_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
+			return value / value2
-	metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.node_stats.indices.indexing.index_total"
+		}
-	metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
+		metricItem.AddLine("Searching", "Fetch Latency", "Average latency for searching fetch.", "group2", "payload.elasticsearch.node_stats.indices.search.fetch_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
-		return value / value2
+		metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.fetch_total"
+		metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
+			return value / value2
+		}
+		metricItem.AddLine("Searching", "Scroll Latency", "Average latency for searching fetch.", "group2", "payload.elasticsearch.node_stats.indices.search.scroll_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
+		metricItem.Lines[2].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.scroll_total"
+		metricItem.Lines[2].Metric.Calc = func(value, value2 float64) float64 {
+			return value / value2
+		}
+		metricItems = append(metricItems, metricItem)
+	default:
+		panic("unknown metric key: " + metricKey)
 	}
-	metricItem.AddLine("Indexing", "Delete Latency", "Average latency for delete documents.", "group1", "payload.elasticsearch.node_stats.indices.indexing.delete_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
-	metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.node_stats.indices.indexing.delete_total"
-	metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
-		return value / value2
-	}
-	metricItems = append(metricItems, metricItem)
-
-	metricItem = newMetricItem("search_latency", 3, LatencyGroupKey)
-	metricItem.AddAxi("searching", "group2", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
-
-	metricItem.AddLine("Searching", "Query Latency", "Average latency for searching query.", "group2", "payload.elasticsearch.node_stats.indices.search.query_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
-	metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.query_total"
-	metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
-		return value / value2
-	}
-	metricItem.AddLine("Searching", "Fetch Latency", "Average latency for searching fetch.", "group2", "payload.elasticsearch.node_stats.indices.search.fetch_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
-	metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.fetch_total"
-	metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
-		return value / value2
-	}
-	metricItem.AddLine("Searching", "Scroll Latency", "Average latency for searching fetch.", "group2", "payload.elasticsearch.node_stats.indices.search.scroll_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
-	metricItem.Lines[2].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.scroll_total"
-	metricItem.Lines[2].Metric.Calc = func(value, value2 float64) float64 {
-		return value / value2
-	}
-	metricItems = append(metricItems, metricItem)
 	query := map[string]interface{}{}
 	clusterUUID, err := adapter.GetClusterUUID(id)
 	if err != nil {
@@ -1040,10 +1127,10 @@ func (h *APIHandler) GetClusterIndexMetrics(id string, bucketSize int, min, max
 			},
 		},
 	}
-	return h.getSingleIndexMetricsByNodeStats(metricItems, query, bucketSize)
+	return h.getSingleIndexMetricsByNodeStats(ctx, metricItems, query, bucketSize)
 }
 
-func (h *APIHandler) getShardsMetric(id string, min, max int64, bucketSize int) map[string]*common.MetricItem {
+func (h *APIHandler) getShardsMetric(ctx context.Context, id string, min, max int64, bucketSize int) map[string]*common.MetricItem {
 	bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
 	query := util.MapStr{
 		"query": util.MapStr{
@@ -1102,10 +1189,10 @@ func (h *APIHandler) getShardsMetric(id string, min, max int64, bucketSize int)
 	metricItem.AddLine("Delayed Unassigned Shards", "Delayed Unassigned Shards", "", "group1", "payload.elasticsearch.cluster_health.delayed_unassigned_shards", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false)
 	var clusterHealthMetrics []*common.MetricItem
 	clusterHealthMetrics = append(clusterHealthMetrics, metricItem)
-	return h.getSingleMetrics(clusterHealthMetrics, query, bucketSize)
+	return h.getSingleMetrics(ctx, clusterHealthMetrics, query, bucketSize)
 }
 
-func (h *APIHandler) getCircuitBreakerMetric(id string, min, max int64, bucketSize int) map[string]*common.MetricItem {
+func (h *APIHandler) getCircuitBreakerMetric(ctx context.Context, id string, min, max int64, bucketSize int) map[string]*common.MetricItem {
 	bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
 	query := util.MapStr{
 		"query": util.MapStr{
@@ -1163,10 +1250,10 @@ func (h *APIHandler) getCircuitBreakerMetric(id string, min, max int64, bucketSi
 	metricItem.AddLine("In Flight Requests Breaker Tripped", "In Flight Requests Tripped", "", "group1", "payload.elasticsearch.node_stats.breakers.in_flight_requests.tripped", "sum", bucketSizeStr, "times/s", "num", "0,0.[00]", "0,0.[00]", false, true)
 	var circuitBreakerMetrics []*common.MetricItem
 	circuitBreakerMetrics = append(circuitBreakerMetrics, metricItem)
-	return h.getSingleMetrics(circuitBreakerMetrics, query, bucketSize)
+	return h.getSingleMetrics(ctx, circuitBreakerMetrics, query, bucketSize)
 }
 
-func (h *APIHandler) getClusterStatusMetric(id string, min, max int64, bucketSize int) (*common.MetricItem, error) {
+func (h *APIHandler) getClusterStatusMetric(ctx context.Context, id string, min, max int64, bucketSize int) (*common.MetricItem, error) {
 	bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
 	intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr)
 	if err != nil {
@@ -1227,7 +1314,8 @@ func (h *APIHandler) getClusterStatusMetric(id string, min, max int64, bucketSiz
 			},
 		},
 	}
-	response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query))
+	queryDSL := util.MustToJSONBytes(query)
+	response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).QueryDSL(ctx, getAllMetricsIndex(), nil, queryDSL)
 	if err != nil {
 		log.Error(err)
 		return nil, err
@@ -1244,6 +1332,7 @@ func (h *APIHandler) getClusterStatusMetric(id string, min, max int64, bucketSiz
 	}
 	metricItem.Lines[0].Data = metricData
 	metricItem.Lines[0].Type = common.GraphTypeBar
+	metricItem.Request = string(queryDSL)
 	return metricItem, nil
 }
 

modules/elastic/api/metrics_util.go

@@ -24,6 +24,7 @@
 package api
 
 import (
+	"context"
 	"fmt"
 	"infini.sh/framework/core/env"
 	"net/http"
@@ -109,9 +110,10 @@ func generateGroupAggs(nodeMetricItems []GroupMetricItem) map[string]interface{}
 	return aggs
 }
 
-func (h *APIHandler) getMetrics(query map[string]interface{}, grpMetricItems []GroupMetricItem, bucketSize int) map[string]*common.MetricItem {
+func (h *APIHandler) getMetrics(ctx context.Context, query map[string]interface{}, grpMetricItems []GroupMetricItem, bucketSize int) map[string]*common.MetricItem {
 	bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
-	response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query))
+	queryDSL := util.MustToJSONBytes(query)
+	response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).QueryDSL(ctx, getAllMetricsIndex(),nil, queryDSL)
 	if err != nil {
 		log.Error(err)
 		panic(err)
@@ -214,6 +216,7 @@ func (h *APIHandler) getMetrics(query map[string]interface{}, grpMetricItems []G
 				line.Data = temp
 			}
 		}
+		metricItem.MetricItem.Request = string(queryDSL)
 		result[metricItem.Key] = metricItem.MetricItem
 	}
 	return result
@@ -337,7 +340,7 @@ func GetMetricRangeAndBucketSize(minStr string, maxStr string, bucketSize int, m
 }
 
 // 获取单个指标，可以包含多条曲线
-func (h *APIHandler) getSingleMetrics(metricItems []*common.MetricItem, query map[string]interface{}, bucketSize int) map[string]*common.MetricItem {
+func (h *APIHandler) getSingleMetrics(ctx context.Context, metricItems []*common.MetricItem, query map[string]interface{}, bucketSize int) map[string]*common.MetricItem {
 	metricData := map[string][][]interface{}{}
 
 	aggs := map[string]interface{}{}
@@ -396,7 +399,8 @@ func (h *APIHandler) getSingleMetrics(metricItems []*common.MetricItem, query ma
 			"aggs": aggs,
 		},
 	}
-	response, err := elastic.GetClient(clusterID).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query))
+	queryDSL := util.MustToJSONBytes(query)
+	response, err := elastic.GetClient(clusterID).QueryDSL(ctx, getAllMetricsIndex(), nil,  queryDSL)
 	if err != nil {
 		log.Error(err)
 		panic(err)
@@ -467,6 +471,7 @@ func (h *APIHandler) getSingleMetrics(metricItems []*common.MetricItem, query ma
 				line.Data = temp
 			}
 		}
+		metricItem.Request = string(queryDSL)
 		result[metricItem.Key] = metricItem
 	}
 
@@ -959,7 +964,7 @@ func parseGroupMetricData(buckets []elastic.BucketBase, isPercent bool) ([]inter
 	return metricData, nil
 }
 
-func (h *APIHandler) getSingleIndexMetricsByNodeStats(metricItems []*common.MetricItem, query map[string]interface{}, bucketSize int) map[string]*common.MetricItem {
+func (h *APIHandler) getSingleIndexMetricsByNodeStats(ctx context.Context, metricItems []*common.MetricItem, query map[string]interface{}, bucketSize int) map[string]*common.MetricItem {
 	metricData := map[string][][]interface{}{}
 
 	aggs := util.MapStr{}
@@ -1041,10 +1046,10 @@ func (h *APIHandler) getSingleIndexMetricsByNodeStats(metricItems []*common.Metr
 			"aggs": sumAggs,
 		},
 	}
-	return parseSingleIndexMetrics(clusterID, metricItems, query, bucketSize,metricData, metricItemsMap)
+	return parseSingleIndexMetrics(ctx, clusterID, metricItems, query, bucketSize,metricData, metricItemsMap)
 }
 
-func (h *APIHandler) getSingleIndexMetrics(metricItems []*common.MetricItem, query map[string]interface{}, bucketSize int) map[string]*common.MetricItem {
+func (h *APIHandler) getSingleIndexMetrics(ctx context.Context, metricItems []*common.MetricItem, query map[string]interface{}, bucketSize int) map[string]*common.MetricItem {
 	metricData := map[string][][]interface{}{}
 
 	aggs := util.MapStr{}
@@ -1146,11 +1151,12 @@ func (h *APIHandler) getSingleIndexMetrics(metricItems []*common.MetricItem, que
 			"aggs": sumAggs,
 		},
 	}
-	return parseSingleIndexMetrics(clusterID, metricItems, query, bucketSize,metricData, metricItemsMap)
+	return parseSingleIndexMetrics(ctx, clusterID, metricItems, query, bucketSize,metricData, metricItemsMap)
 }
 
-func parseSingleIndexMetrics(clusterID string, metricItems []*common.MetricItem, query map[string]interface{}, bucketSize int, metricData map[string][][]interface{}, metricItemsMap map[string]*common.MetricLine) map[string]*common.MetricItem {
+func parseSingleIndexMetrics(ctx context.Context, clusterID string, metricItems []*common.MetricItem, query map[string]interface{}, bucketSize int, metricData map[string][][]interface{}, metricItemsMap map[string]*common.MetricLine) map[string]*common.MetricItem {
-	response, err := elastic.GetClient(clusterID).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query))
+	queryDSL := util.MustToJSONBytes(query)
+	response, err := elastic.GetClient(clusterID).QueryDSL(ctx, getAllMetricsIndex(), nil, util.MustToJSONBytes(query))
 	if err != nil {
 		panic(err)
 	}
@@ -1220,6 +1226,7 @@ func parseSingleIndexMetrics(clusterID string, metricItems []*common.MetricItem,
 				line.Data = temp
 			}
 		}
+		metricItem.Request = string(queryDSL)
 		result[metricItem.Key] = metricItem
 	}
 

modules/elastic/api/node_overview.go

@@ -28,6 +28,7 @@
 package api
 
 import (
+	"context"
 	"fmt"
 	log "github.com/cihub/seelog"
 	httprouter "infini.sh/framework/core/api/router"
@@ -410,7 +411,7 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps
 			},
 		},
 	}
-	metrics := h.getMetrics(query, nodeMetricItems, bucketSize)
+	metrics := h.getMetrics(context.Background(), query, nodeMetricItems, bucketSize)
 	indexMetrics := map[string]util.MapStr{}
 	for key, item := range metrics {
 		for _, line := range item.Lines {
@@ -562,6 +563,12 @@ func (h *APIHandler) GetNodeInfo(w http.ResponseWriter, req *http.Request, ps ht
 	h.WriteJSON(w, kvs, http.StatusOK)
 }
 
+const (
+	NodeCPUJVMMetricKey = "jvm"
+	NodeHealthMetricKey = "node_health"
+	ShardStateMetricKey = "shard_state"
+)
+
 func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
 	clusterID := ps.MustGetParameter("id")
 	clusterUUID, err := adapter.GetClusterUUID(clusterID)
@@ -628,127 +635,155 @@ func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Reque
 
 	bucketSizeStr:=fmt.Sprintf("%vs",bucketSize)
 	metricItems:=[]*common.MetricItem{}
-	metricItem:=newMetricItem("cpu", 1, SystemGroupKey)
+	metricKey := h.GetParameter(req, "key")
-	metricItem.AddAxi("cpu","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true)
+	timeout := h.GetParameterOrDefault(req, "timeout", "60s")
-	metricItem.AddLine("Process CPU","Process CPU","process cpu used percent of node.","group1","payload.elasticsearch.node_stats.process.cpu.percent","max",bucketSizeStr,"%","num","0,0.[00]","0,0.[00]",false,false)
+	du, err := time.ParseDuration(timeout)
-	metricItem.AddLine("OS CPU","OS CPU","process cpu used percent of node.","group1","payload.elasticsearch.node_stats.os.cpu.percent","max",bucketSizeStr,"%","num","0,0.[00]","0,0.[00]",false,false)
-	metricItems=append(metricItems,metricItem)
-	metricItem =newMetricItem("jvm", 2, SystemGroupKey)
-	metricItem.AddAxi("JVM Heap","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
-	metricItem.AddLine("Max Heap","Max Heap","JVM max Heap of node.","group1","payload.elasticsearch.node_stats.jvm.mem.heap_max_in_bytes","max",bucketSizeStr,"","bytes","0,0.[00]","0,0.[00]",false,false)
-	metricItem.AddLine("Used Heap","Used Heap","JVM used Heap of node.","group1","payload.elasticsearch.node_stats.jvm.mem.heap_used_in_bytes","max",bucketSizeStr,"","bytes","0,0.[00]","0,0.[00]",false,false)
-	metricItems=append(metricItems,metricItem)
-	metricItem=newMetricItem("index_throughput", 3, OperationGroupKey)
-	metricItem.AddAxi("indexing","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
-	metricItem.AddLine("Indexing Rate","Total Shards","Number of documents being indexed for node.","group1","payload.elasticsearch.node_stats.indices.indexing.index_total","max",bucketSizeStr,"doc/s","num","0,0.[00]","0,0.[00]",false,true)
-	metricItems=append(metricItems,metricItem)
-	metricItem=newMetricItem("search_throughput", 4, OperationGroupKey)
-	metricItem.AddAxi("searching","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,false)
-	metricItem.AddLine("Search Rate","Total Shards",
-		"Number of search requests being executed.",
-		"group1","payload.elasticsearch.node_stats.indices.search.query_total","max",bucketSizeStr,"query/s","num","0,0.[00]","0,0.[00]",false,true)
-	metricItems=append(metricItems,metricItem)
-
-	metricItem=newMetricItem("index_latency", 5, LatencyGroupKey)
-	metricItem.AddAxi("indexing","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
-
-	metricItem.AddLine("Indexing","Indexing Latency","Average latency for indexing documents.","group1","payload.elasticsearch.node_stats.indices.indexing.index_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true)
-	metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.node_stats.indices.indexing.index_total"
-	metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
-		return value/value2
-	}
-	metricItem.AddLine("Indexing","Delete Latency","Average latency for delete documents.","group1","payload.elasticsearch.node_stats.indices.indexing.delete_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true)
-	metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.node_stats.indices.indexing.delete_total"
-	metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
-		return value/value2
-	}
-	metricItems=append(metricItems,metricItem)
-
-	metricItem=newMetricItem("search_latency", 6, LatencyGroupKey)
-	metricItem.AddAxi("searching","group2",common.PositionLeft,"num","0,0","0,0.[00]",5,false)
-
-	metricItem.AddLine("Searching","Query Latency","Average latency for searching query.","group2","payload.elasticsearch.node_stats.indices.search.query_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true)
-	metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.query_total"
-	metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
-		return value/value2
-	}
-	metricItem.AddLine("Searching","Fetch Latency","Average latency for searching fetch.","group2","payload.elasticsearch.node_stats.indices.search.fetch_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true)
-	metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.fetch_total"
-	metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
-		return value/value2
-	}
-	metricItem.AddLine("Searching","Scroll Latency","Average latency for searching fetch.","group2","payload.elasticsearch.node_stats.indices.search.scroll_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true)
-	metricItem.Lines[2].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.scroll_total"
-	metricItem.Lines[2].Metric.Calc = func(value, value2 float64) float64 {
-		return value/value2
-	}
-	metricItems=append(metricItems,metricItem)
-	metricItem =newMetricItem("parent_breaker", 8, SystemGroupKey)
-	metricItem.AddLine("Parent Breaker Tripped","Parent Breaker Tripped","Rate of the circuit breaker has been triggered and prevented an out of memory error.","group1","payload.elasticsearch.node_stats.breakers.parent.tripped","max",bucketSizeStr,"times/s","num","0,0.[00]","0,0.[00]",false,true)
-	metricItems=append(metricItems,metricItem)
-	metrics := h.getSingleMetrics(metricItems,query, bucketSize)
-	healthMetric, err := getNodeHealthMetric(query, bucketSize)
 	if err != nil {
 		log.Error(err)
+		h.WriteError(w, err.Error(), http.StatusInternalServerError)
+		return
 	}
-	query = util.MapStr{
+	ctx, cancel := context.WithTimeout(context.Background(), du)
-		"size": 0,
+	defer cancel()
-		"query": util.MapStr{
+	metrics := map[string]*common.MetricItem{}
-			"bool": util.MapStr{
+	if metricKey == NodeHealthMetricKey {
-				"must": []util.MapStr{
+		healthMetric, err := getNodeHealthMetric(ctx, query, bucketSize)
-					{
+		if err != nil {
-						"term":util.MapStr{
+			log.Error(err)
-							"metadata.labels.cluster_uuid":util.MapStr{
+			h.WriteError(w, err.Error(), http.StatusInternalServerError)
-								"value": clusterUUID,
+			return
+		}
+		metrics["node_health"] = healthMetric
+	}else if metricKey == ShardStateMetricKey {
+		query = util.MapStr{
+			"size": 0,
+			"query": util.MapStr{
+				"bool": util.MapStr{
+					"must": []util.MapStr{
+						{
+							"term":util.MapStr{
+								"metadata.labels.cluster_uuid":util.MapStr{
+									"value": clusterUUID,
+								},
+							},
+						},
+						{
+							"term": util.MapStr{
+								"metadata.category": util.MapStr{
+									"value": "elasticsearch",
+								},
+							},
+						},
+						{
+							"term": util.MapStr{
+								"metadata.name": util.MapStr{
+									"value": "shard_stats",
+								},
+							},
+						},
+						{
+							"term": util.MapStr{
+								"metadata.labels.node_id": util.MapStr{
+									"value": nodeID,
+								},
 							},
 						},
 					},
-					{
+					"filter": []util.MapStr{
-						"term": util.MapStr{
+						{
-							"metadata.category": util.MapStr{
+							"range": util.MapStr{
-								"value": "elasticsearch",
+								"timestamp": util.MapStr{
-							},
+									"gte": min,
-						},
+									"lte": max,
-					},
+								},
-					{
-						"term": util.MapStr{
-							"metadata.name": util.MapStr{
-								"value": "shard_stats",
-							},
-						},
-					},
-					{
-						"term": util.MapStr{
-							"metadata.labels.node_id": util.MapStr{
-								"value": nodeID,
-							},
-						},
-					},
-				},
-				"filter": []util.MapStr{
-					{
-						"range": util.MapStr{
-							"timestamp": util.MapStr{
-								"gte": min,
-								"lte": max,
 							},
 						},
 					},
 				},
 			},
-		},
+		}
+		shardStateMetric, err := getNodeShardStateMetric(ctx, query, bucketSize)
+		if err != nil {
+			log.Error(err)
+			h.WriteError(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		metrics["shard_state"] = shardStateMetric
+	}else{
+		switch metricKey {
+		case NodeProcessCPUMetricKey:
+			metricItem:=newMetricItem("cpu", 1, SystemGroupKey)
+			metricItem.AddAxi("cpu","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true)
+			metricItem.AddLine("Process CPU","Process CPU","process cpu used percent of node.","group1","payload.elasticsearch.node_stats.process.cpu.percent","max",bucketSizeStr,"%","num","0,0.[00]","0,0.[00]",false,false)
+			metricItem.AddLine("OS CPU","OS CPU","process cpu used percent of node.","group1","payload.elasticsearch.node_stats.os.cpu.percent","max",bucketSizeStr,"%","num","0,0.[00]","0,0.[00]",false,false)
+			metricItems=append(metricItems,metricItem)
+		case NodeCPUJVMMetricKey:
+			metricItem := newMetricItem("jvm", 2, SystemGroupKey)
+			metricItem.AddAxi("JVM Heap","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
+			metricItem.AddLine("Max Heap","Max Heap","JVM max Heap of node.","group1","payload.elasticsearch.node_stats.jvm.mem.heap_max_in_bytes","max",bucketSizeStr,"","bytes","0,0.[00]","0,0.[00]",false,false)
+			metricItem.AddLine("Used Heap","Used Heap","JVM used Heap of node.","group1","payload.elasticsearch.node_stats.jvm.mem.heap_used_in_bytes","max",bucketSizeStr,"","bytes","0,0.[00]","0,0.[00]",false,false)
+			metricItems=append(metricItems,metricItem)
+		case IndexThroughputMetricKey:
+			metricItem := newMetricItem("index_throughput", 3, OperationGroupKey)
+			metricItem.AddAxi("indexing","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
+			metricItem.AddLine("Indexing Rate","Total Shards","Number of documents being indexed for node.","group1","payload.elasticsearch.node_stats.indices.indexing.index_total","max",bucketSizeStr,"doc/s","num","0,0.[00]","0,0.[00]",false,true)
+			metricItems=append(metricItems,metricItem)
+		case SearchThroughputMetricKey:
+			metricItem := newMetricItem("search_throughput", 4, OperationGroupKey)
+			metricItem.AddAxi("searching","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,false)
+			metricItem.AddLine("Search Rate","Total Shards",
+				"Number of search requests being executed.",
+				"group1","payload.elasticsearch.node_stats.indices.search.query_total","max",bucketSizeStr,"query/s","num","0,0.[00]","0,0.[00]",false,true)
+			metricItems=append(metricItems,metricItem)
+		case IndexLatencyMetricKey:
+			metricItem := newMetricItem("index_latency", 5, LatencyGroupKey)
+			metricItem.AddAxi("indexing","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
+
+			metricItem.AddLine("Indexing","Indexing Latency","Average latency for indexing documents.","group1","payload.elasticsearch.node_stats.indices.indexing.index_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true)
+			metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.node_stats.indices.indexing.index_total"
+			metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
+				return value/value2
+			}
+			metricItem.AddLine("Indexing","Delete Latency","Average latency for delete documents.","group1","payload.elasticsearch.node_stats.indices.indexing.delete_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true)
+			metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.node_stats.indices.indexing.delete_total"
+			metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
+				return value/value2
+			}
+			metricItems=append(metricItems,metricItem)
+		case SearchLatencyMetricKey:
+			metricItem := newMetricItem("search_latency", 6, LatencyGroupKey)
+			metricItem.AddAxi("searching","group2",common.PositionLeft,"num","0,0","0,0.[00]",5,false)
+
+			metricItem.AddLine("Searching","Query Latency","Average latency for searching query.","group2","payload.elasticsearch.node_stats.indices.search.query_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true)
+			metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.query_total"
+			metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
+				return value/value2
+			}
+			metricItem.AddLine("Searching","Fetch Latency","Average latency for searching fetch.","group2","payload.elasticsearch.node_stats.indices.search.fetch_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true)
+			metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.fetch_total"
+			metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
+				return value/value2
+			}
+			metricItem.AddLine("Searching","Scroll Latency","Average latency for searching fetch.","group2","payload.elasticsearch.node_stats.indices.search.scroll_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true)
+			metricItem.Lines[2].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.scroll_total"
+			metricItem.Lines[2].Metric.Calc = func(value, value2 float64) float64 {
+				return value/value2
+			}
+			metricItems=append(metricItems,metricItem)
+		case ParentBreakerMetricKey:
+			metricItem := newMetricItem("parent_breaker", 8, SystemGroupKey)
+			metricItem.AddLine("Parent Breaker Tripped","Parent Breaker Tripped","Rate of the circuit breaker has been triggered and prevented an out of memory error.","group1","payload.elasticsearch.node_stats.breakers.parent.tripped","max",bucketSizeStr,"times/s","num","0,0.[00]","0,0.[00]",false,true)
+			metricItems=append(metricItems,metricItem)
+		}
+
+		metrics = h.getSingleMetrics(ctx, metricItems,query, bucketSize)
 	}
-	shardStateMetric, err := getNodeShardStateMetric(query, bucketSize)
+
-	if err != nil {
-		log.Error(err)
-	}
-	metrics["node_health"] = healthMetric
-	metrics["shard_state"] = shardStateMetric
 	resBody["metrics"] = metrics
 	h.WriteJSON(w, resBody, http.StatusOK)
 }
 
-func getNodeShardStateMetric(query util.MapStr, bucketSize int)(*common.MetricItem, error){
+func getNodeShardStateMetric(ctx context.Context, query util.MapStr, bucketSize int)(*common.MetricItem, error){
 	bucketSizeStr:=fmt.Sprintf("%vs",bucketSize)
 	intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr)
 	if err != nil {
@@ -771,7 +806,8 @@ func getNodeShardStateMetric(query util.MapStr, bucketSize int)(*common.MetricIt
 			},
 		},
 	}
-	response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query))
+	queryDSL := util.MustToJSONBytes(query)
+	response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).QueryDSL(ctx, getAllMetricsIndex(), nil, queryDSL)
 	if err != nil {
 		log.Error(err)
 		return nil, err
@@ -789,10 +825,11 @@ func getNodeShardStateMetric(query util.MapStr, bucketSize int)(*common.MetricIt
 	}
 	metricItem.Lines[0].Data = metricData
 	metricItem.Lines[0].Type = common.GraphTypeBar
+	metricItem.Request = string(queryDSL)
 	return metricItem, nil
 }
 
-func getNodeHealthMetric(query util.MapStr, bucketSize int)(*common.MetricItem, error){
+func getNodeHealthMetric(ctx context.Context, query util.MapStr, bucketSize int)(*common.MetricItem, error){
 	bucketSizeStr:=fmt.Sprintf("%vs",bucketSize)
 	intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr)
 	if err != nil {
@@ -813,7 +850,8 @@ func getNodeHealthMetric(query util.MapStr, bucketSize int)(*common.MetricItem,
 			},
 		},
 	}
-	response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query))
+	queryDSL := util.MustToJSONBytes(query)
+	response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).QueryDSL(ctx, getAllMetricsIndex(), nil,  queryDSL)
 	if err != nil {
 		log.Error(err)
 		return nil, err
@@ -844,6 +882,7 @@ func getNodeHealthMetric(query util.MapStr, bucketSize int)(*common.MetricItem,
 			}
 		}
 	}
+	metricItem.Request = string(queryDSL)
 	metricItem.Lines[0].Data = metricData
 	metricItem.Lines[0].Type = common.GraphTypeBar
 	return metricItem, nil

modules/elastic/api/threadpool_metrics.go

@@ -24,6 +24,7 @@
 package api
 
 import (
+	"context"
 	"fmt"
 	log "github.com/cihub/seelog"
 	"infini.sh/framework/core/elastic"
@@ -45,7 +46,42 @@ const (
 	ThreadPoolBulkGroupKey  = "thread_pool_bulk"
 )
 
-func (h *APIHandler) getThreadPoolMetrics(clusterID string, bucketSize int, min, max int64, nodeName string, top int) (map[string]*common.MetricItem, error){
+const (
+	SearchThreadsMetricKey = "search_threads"
+	IndexThreadsMetricKey  = "index_threads"
+	BulkThreadsMetricKey  = "bulk_threads"
+	FlushThreadsMetricKey = "flush_threads"
+	RefreshThreadsMetricKey  = "refresh_threads"
+	WriteThreadsMetricKey = "write_threads"
+	ForceMergeThreadsMetricKey = "force_merge_threads"
+	SearchQueueMetricKey = "search_queue"
+	IndexQueueMetricKey  = "index_queue"
+	BulkQueueMetricKey  = "bulk_queue"
+	FlushQueueMetricKey = "flush_queue"
+	RefreshQueueMetricKey  = "refresh_queue"
+	WriteQueueMetricKey = "write_queue"
+	SearchActiveMetricKey = "search_active"
+	IndexActiveMetricKey  = "index_active"
+	BulkActiveMetricKey  = "bulk_active"
+	FlushActiveMetricKey = "flush_active"
+	WriteActiveMetricKey = "write_active"
+	ForceMergeActiveMetricKey = "force_merge_active"
+	SearchRejectedMetricKey = "search_rejected"
+	IndexRejectedMetricKey  = "index_rejected"
+	BulkRejectedMetricKey  = "bulk_rejected"
+	FlushRejectedMetricKey = "flush_rejected"
+	WriteRejectedMetricKey = "write_rejected"
+	ForceMergeRejectedMetricKey = "force_merge_rejected"
+	GetThreadsMetricKey = "get_threads"
+	GetQueueMetricKey = "get_queue"
+	GetActiveMetricKey = "get_active"
+	GetRejectedMetricKey = "get_rejected"
+	RefreshActiveMetricKey = "refresh_active"
+	RefreshRejectedMetricKey = "refresh_rejected"
+	ForceMergeQueueMetricKey = "force_merge_queue"
+)
+
+func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string, bucketSize int, min, max int64, nodeName string, top int, metricKey string) (map[string]*common.MetricItem, error){
 	clusterUUID, err := adapter.GetClusterUUID(clusterID)
 	if err != nil {
 		return nil, err
@@ -123,11 +159,12 @@ func (h *APIHandler) getThreadPoolMetrics(clusterID string, bucketSize int, min,
 			},
 		},
 	}
-	searchThreadsMetric := newMetricItem("search_threads", 1, ThreadPoolSearchGroupKey)
+	queueMetricItems := []GroupMetricItem{}
-	searchThreadsMetric.AddAxi("Search Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+	switch metricKey {
-
+	case SearchThreadsMetricKey:
-	queueMetricItems := []GroupMetricItem{
+		searchThreadsMetric := newMetricItem(SearchThreadsMetricKey, 1, ThreadPoolSearchGroupKey)
-		{
+		searchThreadsMetric.AddAxi("Search Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		queueMetricItems = append(queueMetricItems, GroupMetricItem{
 			Key: "search_threads",
 			Field: "payload.elasticsearch.node_stats.thread_pool.search.threads",
 			ID: util.GetUUID(),
@@ -135,148 +172,153 @@ func (h *APIHandler) getThreadPoolMetrics(clusterID string, bucketSize int, min,
 			MetricItem: searchThreadsMetric,
 			FormatType: "num",
 			Units: "",
-		},
+		})
-	}
+	case SearchQueueMetricKey:
-	searchQueueMetric := newMetricItem("search_queue", 1, ThreadPoolSearchGroupKey)
+		searchQueueMetric := newMetricItem(SearchQueueMetricKey, 1, ThreadPoolSearchGroupKey)
-	searchQueueMetric.AddAxi("Search Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		searchQueueMetric.AddAxi("Search Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
 
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
+		queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "search_queue",
+			Key: "search_queue",
-		Field: "payload.elasticsearch.node_stats.thread_pool.search.queue",
+			Field: "payload.elasticsearch.node_stats.thread_pool.search.queue",
-		ID: util.GetUUID(),
+			ID: util.GetUUID(),
-		IsDerivative: false,
+			IsDerivative: false,
-		MetricItem: searchQueueMetric,
+			MetricItem: searchQueueMetric,
-		FormatType: "num",
+			FormatType: "num",
-		Units: "",
+			Units: "",
-	})
+		})
-	searchActiveMetric := newMetricItem("search_active", 1, ThreadPoolSearchGroupKey)
+	case SearchActiveMetricKey:
-	searchActiveMetric.AddAxi("Search Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		searchActiveMetric := newMetricItem(SearchActiveMetricKey, 1, ThreadPoolSearchGroupKey)
+		searchActiveMetric.AddAxi("Search Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
 
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
+		queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "search_active",
+			Key: "search_active",
-		Field: "payload.elasticsearch.node_stats.thread_pool.search.active",
+			Field: "payload.elasticsearch.node_stats.thread_pool.search.active",
-		ID: util.GetUUID(),
+			ID: util.GetUUID(),
-		IsDerivative: false,
+			IsDerivative: false,
-		MetricItem: searchActiveMetric,
+			MetricItem: searchActiveMetric,
-		FormatType: "num",
+			FormatType: "num",
-		Units: "",
+			Units: "",
-	})
+		})
-	searchRejectedMetric := newMetricItem("search_rejected", 1, ThreadPoolSearchGroupKey)
+	case SearchRejectedMetricKey:
-	searchRejectedMetric.AddAxi("Search Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		searchRejectedMetric := newMetricItem(SearchRejectedMetricKey, 1, ThreadPoolSearchGroupKey)
+		searchRejectedMetric.AddAxi("Search Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
 
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
+		queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "search_rejected",
+			Key: "search_rejected",
-		Field: "payload.elasticsearch.node_stats.thread_pool.search.rejected",
+			Field: "payload.elasticsearch.node_stats.thread_pool.search.rejected",
-		ID: util.GetUUID(),
+			ID: util.GetUUID(),
-		IsDerivative: true,
+			IsDerivative: true,
-		MetricItem: searchRejectedMetric,
+			MetricItem: searchRejectedMetric,
-		FormatType: "num",
+			FormatType: "num",
-		Units: "rejected/s",
+			Units: "rejected/s",
-	})
+		})
+	case GetThreadsMetricKey:
+		getThreadsMetric := newMetricItem(GetThreadsMetricKey, 1, ThreadPoolGetGroupKey)
+		getThreadsMetric.AddAxi("Get Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
 
-	getThreadsMetric := newMetricItem("get_threads", 1, ThreadPoolGetGroupKey)
+		queueMetricItems = append(queueMetricItems, GroupMetricItem{
-	getThreadsMetric.AddAxi("Get Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+			Key: "get_threads",
+			Field: "payload.elasticsearch.node_stats.thread_pool.get.threads",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: getThreadsMetric,
+			FormatType: "num",
+			Units: "",
+		})
+	case GetQueueMetricKey:
+		getQueueMetric := newMetricItem(GetQueueMetricKey, 1, ThreadPoolGetGroupKey)
+		getQueueMetric.AddAxi("Get Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
 
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
+		queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "get_threads",
+			Key: "get_queue",
-		Field: "payload.elasticsearch.node_stats.thread_pool.get.threads",
+			Field: "payload.elasticsearch.node_stats.thread_pool.get.queue",
-		ID: util.GetUUID(),
+			ID: util.GetUUID(),
-		IsDerivative: false,
+			IsDerivative: false,
-		MetricItem: getThreadsMetric,
+			MetricItem: getQueueMetric,
-		FormatType: "num",
+			FormatType: "num",
-		Units: "",
+			Units: "",
-	})
+		})
-	getQueueMetric := newMetricItem("get_queue", 1, ThreadPoolGetGroupKey)
+	case GetActiveMetricKey:
-	getQueueMetric.AddAxi("Get Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		getActiveMetric := newMetricItem(GetActiveMetricKey, 1, ThreadPoolGetGroupKey)
+		getActiveMetric.AddAxi("Get Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
 
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
+		queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "get_queue",
+			Key: "get_active",
-		Field: "payload.elasticsearch.node_stats.thread_pool.get.queue",
+			Field: "payload.elasticsearch.node_stats.thread_pool.get.active",
-		ID: util.GetUUID(),
+			ID: util.GetUUID(),
-		IsDerivative: false,
+			IsDerivative: false,
-		MetricItem: getQueueMetric,
+			MetricItem: getActiveMetric,
-		FormatType: "num",
+			FormatType: "num",
-		Units: "",
+			Units: "",
-	})
+		})
-	getActiveMetric := newMetricItem("get_active", 1, ThreadPoolGetGroupKey)
+	case GetRejectedMetricKey:
-	getActiveMetric.AddAxi("Get Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		getRejectedMetric := newMetricItem(GetRejectedMetricKey, 1, ThreadPoolGetGroupKey)
+		getRejectedMetric.AddAxi("Get Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
 
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
+		queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "get_active",
+			Key: "get_rejected",
-		Field: "payload.elasticsearch.node_stats.thread_pool.get.active",
+			Field: "payload.elasticsearch.node_stats.thread_pool.get.rejected",
-		ID: util.GetUUID(),
+			ID: util.GetUUID(),
-		IsDerivative: false,
+			IsDerivative: true,
-		MetricItem: getActiveMetric,
+			MetricItem: getRejectedMetric,
-		FormatType: "num",
+			FormatType: "num",
-		Units: "",
+			Units: "rejected/s",
-	})
+		})
-	getRejectedMetric := newMetricItem("get_rejected", 1, ThreadPoolGetGroupKey)
+	case FlushThreadsMetricKey:
-	getRejectedMetric.AddAxi("Get Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		flushThreadsMetric := newMetricItem(FlushThreadsMetricKey, 1, ThreadPoolFlushGroupKey)
+		flushThreadsMetric.AddAxi("Flush Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
 
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
+		queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "get_rejected",
+			Key: "flush_threads",
-		Field: "payload.elasticsearch.node_stats.thread_pool.get.rejected",
+			Field: "payload.elasticsearch.node_stats.thread_pool.flush.threads",
-		ID: util.GetUUID(),
+			ID: util.GetUUID(),
-		IsDerivative: true,
+			IsDerivative: false,
-		MetricItem: getRejectedMetric,
+			MetricItem: flushThreadsMetric,
-		FormatType: "num",
+			FormatType: "num",
-		Units: "rejected/s",
+			Units: "",
-	})
+		})
+	case FlushQueueMetricKey:
+		flushQueueMetric := newMetricItem(FlushQueueMetricKey, 1, ThreadPoolFlushGroupKey)
+		flushQueueMetric.AddAxi("Get Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
 
-	flushThreadsMetric := newMetricItem("flush_threads", 1, ThreadPoolFlushGroupKey)
+		queueMetricItems = append(queueMetricItems, GroupMetricItem{
-	flushThreadsMetric.AddAxi("Flush Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+			Key: "flush_queue",
+			Field: "payload.elasticsearch.node_stats.thread_pool.flush.queue",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: flushQueueMetric,
+			FormatType: "num",
+			Units: "",
+		})
+	case FlushActiveMetricKey:
+		flushActiveMetric := newMetricItem(FlushActiveMetricKey, 1, ThreadPoolFlushGroupKey)
+		flushActiveMetric.AddAxi("Flush Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
 
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
+		queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "flush_threads",
+			Key: "flush_active",
-		Field: "payload.elasticsearch.node_stats.thread_pool.flush.threads",
+			Field: "payload.elasticsearch.node_stats.thread_pool.flush.active",
-		ID: util.GetUUID(),
+			ID: util.GetUUID(),
-		IsDerivative: false,
+			IsDerivative: false,
-		MetricItem: flushThreadsMetric,
+			MetricItem: flushActiveMetric,
-		FormatType: "num",
+			FormatType: "num",
-		Units: "",
+			Units: "",
-	})
+		})
-	flushQueueMetric := newMetricItem("flush_queue", 1, ThreadPoolFlushGroupKey)
-	flushQueueMetric.AddAxi("Get Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
 
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
+	case FlushRejectedMetricKey:
-		Key: "flush_queue",
+		flushRejectedMetric := newMetricItem(FlushRejectedMetricKey, 1, ThreadPoolFlushGroupKey)
-		Field: "payload.elasticsearch.node_stats.thread_pool.flush.queue",
+		flushRejectedMetric.AddAxi("Flush Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: flushQueueMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	flushActiveMetric := newMetricItem("flush_active", 1, ThreadPoolFlushGroupKey)
-	flushActiveMetric.AddAxi("Flush Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
 
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
+		queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "flush_active",
+			Key: "flush_rejected",
-		Field: "payload.elasticsearch.node_stats.thread_pool.flush.active",
+			Field: "payload.elasticsearch.node_stats.thread_pool.flush.rejected",
-		ID: util.GetUUID(),
+			ID: util.GetUUID(),
-		IsDerivative: false,
+			IsDerivative: true,
-		MetricItem: flushActiveMetric,
+			MetricItem: flushRejectedMetric,
-		FormatType: "num",
+			FormatType: "num",
-		Units: "",
+			Units: "rejected/s",
-	})
+		})
-	flushRejectedMetric := newMetricItem("flush_rejected", 1, ThreadPoolFlushGroupKey)
+	case IndexThreadsMetricKey:
-	flushRejectedMetric.AddAxi("Flush Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		indexThreadsMetric := newMetricItem(IndexThreadsMetricKey, 1, ThreadPoolIndexGroupKey)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "flush_rejected",
-		Field: "payload.elasticsearch.node_stats.thread_pool.flush.rejected",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: flushRejectedMetric,
-		FormatType: "num",
-		Units: "rejected/s",
-	})
-
-	majorVersion := elastic.GetMetadata(clusterID).GetMajorVersion()
-	ver := elastic.GetClient(clusterID).GetVersion()
-
-	if (ver.Distribution == "" || ver.Distribution == elastic.Elasticsearch) && majorVersion < 6{
-		indexThreadsMetric := newMetricItem("index_threads", 1, ThreadPoolIndexGroupKey)
 		indexThreadsMetric.AddAxi("Index Threads Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
 
 		queueMetricItems = append(queueMetricItems, GroupMetricItem{
@@ -288,7 +330,8 @@ func (h *APIHandler) getThreadPoolMetrics(clusterID string, bucketSize int, min,
 			FormatType:   "num",
 			Units:        "",
 		})
-		indexQueueMetric := newMetricItem("index_queue", 1, ThreadPoolIndexGroupKey)
+	case IndexQueueMetricKey:
+		indexQueueMetric := newMetricItem(IndexQueueMetricKey, 1, ThreadPoolIndexGroupKey)
 		indexQueueMetric.AddAxi("Index Queue Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
 
 		queueMetricItems = append(queueMetricItems, GroupMetricItem{
@@ -300,7 +343,8 @@ func (h *APIHandler) getThreadPoolMetrics(clusterID string, bucketSize int, min,
 			FormatType:   "num",
 			Units:        "",
 		})
-		indexActiveMetric := newMetricItem("index_active", 1, ThreadPoolIndexGroupKey)
+	case IndexActiveMetricKey:
+		indexActiveMetric := newMetricItem(IndexActiveMetricKey, 1, ThreadPoolIndexGroupKey)
 		indexActiveMetric.AddAxi("Index Active Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
 
 		queueMetricItems = append(queueMetricItems, GroupMetricItem{
@@ -312,7 +356,8 @@ func (h *APIHandler) getThreadPoolMetrics(clusterID string, bucketSize int, min,
 			FormatType:   "num",
 			Units:        "",
 		})
-		indexRejectedMetric := newMetricItem("index_rejected", 1, ThreadPoolIndexGroupKey)
+	case IndexRejectedMetricKey:
+		indexRejectedMetric := newMetricItem(IndexRejectedMetricKey, 1, ThreadPoolIndexGroupKey)
 		indexRejectedMetric.AddAxi("Index Rejected Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
 
 		queueMetricItems = append(queueMetricItems, GroupMetricItem{
@@ -324,8 +369,8 @@ func (h *APIHandler) getThreadPoolMetrics(clusterID string, bucketSize int, min,
 			FormatType:   "num",
 			Units:        "rejected/s",
 		})
-
+	case BulkThreadsMetricKey:
-		bulkThreadsMetric := newMetricItem("bulk_threads", 1, ThreadPoolBulkGroupKey)
+		bulkThreadsMetric := newMetricItem(BulkThreadsMetricKey, 1, ThreadPoolBulkGroupKey)
 		bulkThreadsMetric.AddAxi("Bulk Threads Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
 
 		queueMetricItems = append(queueMetricItems, GroupMetricItem{
@@ -337,7 +382,8 @@ func (h *APIHandler) getThreadPoolMetrics(clusterID string, bucketSize int, min,
 			FormatType:   "num",
 			Units:        "",
 		})
-		bulkQueueMetric := newMetricItem("bulk_queue", 1, ThreadPoolBulkGroupKey)
+	case BulkQueueMetricKey:
+		bulkQueueMetric := newMetricItem(BulkQueueMetricKey, 1, ThreadPoolBulkGroupKey)
 		bulkQueueMetric.AddAxi("Bulk Queue Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
 
 		queueMetricItems = append(queueMetricItems, GroupMetricItem{
@@ -349,7 +395,8 @@ func (h *APIHandler) getThreadPoolMetrics(clusterID string, bucketSize int, min,
 			FormatType:   "num",
 			Units:        "",
 		})
-		bulkActiveMetric := newMetricItem("bulk_active", 1, ThreadPoolBulkGroupKey)
+	case BulkActiveMetricKey:
+		bulkActiveMetric := newMetricItem(BulkActiveMetricKey, 1, ThreadPoolBulkGroupKey)
 		bulkActiveMetric.AddAxi("Bulk Active Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
 
 		queueMetricItems = append(queueMetricItems, GroupMetricItem{
@@ -361,7 +408,8 @@ func (h *APIHandler) getThreadPoolMetrics(clusterID string, bucketSize int, min,
 			FormatType:   "num",
 			Units:        "",
 		})
-		bulkRejectedMetric := newMetricItem("bulk_rejected", 1, ThreadPoolBulkGroupKey)
+	case BulkRejectedMetricKey:
+		bulkRejectedMetric := newMetricItem(BulkRejectedMetricKey, 1, ThreadPoolBulkGroupKey)
 		bulkRejectedMetric.AddAxi("Bulk Rejected Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
 
 		queueMetricItems = append(queueMetricItems, GroupMetricItem{
@@ -373,8 +421,8 @@ func (h *APIHandler) getThreadPoolMetrics(clusterID string, bucketSize int, min,
 			FormatType:   "num",
 			Units:        "rejected/s",
 		})
-	}else {
+	case WriteThreadsMetricKey:
-		writeThreadsMetric := newMetricItem("write_threads", 1, ThreadPoolWriteGroupKey)
+		writeThreadsMetric := newMetricItem(WriteThreadsMetricKey, 1, ThreadPoolWriteGroupKey)
 		writeThreadsMetric.AddAxi("Write Threads Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
 
 		queueMetricItems = append(queueMetricItems, GroupMetricItem{
@@ -386,7 +434,8 @@ func (h *APIHandler) getThreadPoolMetrics(clusterID string, bucketSize int, min,
 			FormatType:   "num",
 			Units:        "",
 		})
-		writeQueueMetric := newMetricItem("write_queue", 1, ThreadPoolWriteGroupKey)
+	case WriteQueueMetricKey:
+		writeQueueMetric := newMetricItem(WriteQueueMetricKey, 1, ThreadPoolWriteGroupKey)
 		writeQueueMetric.AddAxi("Write Queue Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
 
 		queueMetricItems = append(queueMetricItems, GroupMetricItem{
@@ -398,7 +447,8 @@ func (h *APIHandler) getThreadPoolMetrics(clusterID string, bucketSize int, min,
 			FormatType:   "num",
 			Units:        "",
 		})
-		writeActiveMetric := newMetricItem("write_active", 1, ThreadPoolWriteGroupKey)
+	case WriteActiveMetricKey:
+		writeActiveMetric := newMetricItem(WriteActiveMetricKey, 1, ThreadPoolWriteGroupKey)
 		writeActiveMetric.AddAxi("Write Active Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
 
 		queueMetricItems = append(queueMetricItems, GroupMetricItem{
@@ -410,7 +460,8 @@ func (h *APIHandler) getThreadPoolMetrics(clusterID string, bucketSize int, min,
 			FormatType:   "num",
 			Units:        "",
 		})
-		writeRejectedMetric := newMetricItem("write_rejected", 1, ThreadPoolWriteGroupKey)
+	case WriteRejectedMetricKey:
+		writeRejectedMetric := newMetricItem(WriteRejectedMetricKey, 1, ThreadPoolWriteGroupKey)
 		writeRejectedMetric.AddAxi("Write Rejected Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
 
 		queueMetricItems = append(queueMetricItems, GroupMetricItem{
@@ -422,103 +473,113 @@ func (h *APIHandler) getThreadPoolMetrics(clusterID string, bucketSize int, min,
 			FormatType:   "num",
 			Units:        "rejected/s",
 		})
+	case RefreshThreadsMetricKey:
+		refreshThreadsMetric := newMetricItem(RefreshThreadsMetricKey, 1, ThreadPoolRefreshGroupKey)
+		refreshThreadsMetric.AddAxi("Refresh Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+
+		queueMetricItems = append(queueMetricItems, GroupMetricItem{
+			Key: "refresh_threads",
+			Field: "payload.elasticsearch.node_stats.thread_pool.refresh.threads",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: refreshThreadsMetric,
+			FormatType: "num",
+			Units: "",
+		})
+	case RefreshQueueMetricKey:
+		refreshQueueMetric := newMetricItem(RefreshQueueMetricKey, 1, ThreadPoolRefreshGroupKey)
+		refreshQueueMetric.AddAxi("Refresh Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+
+		queueMetricItems = append(queueMetricItems, GroupMetricItem{
+			Key: "refresh_queue",
+			Field: "payload.elasticsearch.node_stats.thread_pool.refresh.queue",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: refreshQueueMetric,
+			FormatType: "num",
+			Units: "",
+		})
+	case RefreshActiveMetricKey:
+		refreshActiveMetric := newMetricItem(RefreshActiveMetricKey, 1, ThreadPoolRefreshGroupKey)
+		refreshActiveMetric.AddAxi("Refresh Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+
+		queueMetricItems = append(queueMetricItems, GroupMetricItem{
+			Key: "refresh_active",
+			Field: "payload.elasticsearch.node_stats.thread_pool.refresh.active",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: refreshActiveMetric,
+			FormatType: "num",
+			Units: "",
+		})
+	case RefreshRejectedMetricKey:
+		refreshRejectedMetric := newMetricItem(RefreshRejectedMetricKey, 1, ThreadPoolRefreshGroupKey)
+		refreshRejectedMetric.AddAxi("Refresh Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+
+		queueMetricItems = append(queueMetricItems, GroupMetricItem{
+			Key: "refresh_rejected",
+			Field: "payload.elasticsearch.node_stats.thread_pool.refresh.rejected",
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: refreshRejectedMetric,
+			FormatType: "num",
+			Units: "rejected/s",
+		})
+	case ForceMergeThreadsMetricKey:
+		forceMergeThreadsMetric := newMetricItem(ForceMergeThreadsMetricKey, 1, ThreadPoolForceMergeGroupKey)
+		forceMergeThreadsMetric.AddAxi("Force Merge Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+
+		queueMetricItems = append(queueMetricItems, GroupMetricItem{
+			Key: "force_merge_threads",
+			Field: "payload.elasticsearch.node_stats.thread_pool.force_merge.threads",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: forceMergeThreadsMetric,
+			FormatType: "num",
+			Units: "",
+		})
+	case ForceMergeQueueMetricKey:
+		forceMergeQueueMetric := newMetricItem(ForceMergeQueueMetricKey, 1, ThreadPoolForceMergeGroupKey)
+		forceMergeQueueMetric.AddAxi("Force Merge Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+
+		queueMetricItems = append(queueMetricItems, GroupMetricItem{
+			Key: "force_merge_queue",
+			Field: "payload.elasticsearch.node_stats.thread_pool.force_merge.queue",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: forceMergeQueueMetric,
+			FormatType: "num",
+			Units: "",
+		})
+	case ForceMergeActiveMetricKey:
+		forceMergeActiveMetric := newMetricItem(ForceMergeActiveMetricKey, 1, ThreadPoolForceMergeGroupKey)
+		forceMergeActiveMetric.AddAxi("Force Merge Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+
+		queueMetricItems = append(queueMetricItems, GroupMetricItem{
+			Key: "force_merge_active",
+			Field: "payload.elasticsearch.node_stats.thread_pool.force_merge.active",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: forceMergeActiveMetric,
+			FormatType: "num",
+			Units: "",
+		})
+	case ForceMergeRejectedMetricKey:
+		forceMergeRejectedMetric := newMetricItem(ForceMergeRejectedMetricKey, 1, ThreadPoolForceMergeGroupKey)
+		forceMergeRejectedMetric.AddAxi("Force Merge Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+
+		queueMetricItems = append(queueMetricItems, GroupMetricItem{
+			Key: "force_merge_rejected",
+			Field: "payload.elasticsearch.node_stats.thread_pool.force_merge.rejected",
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: forceMergeRejectedMetric,
+			FormatType: "num",
+			Units: "rejected/s",
+		})
 	}
-	refreshThreadsMetric := newMetricItem("refresh_threads", 1, ThreadPoolRefreshGroupKey)
-	refreshThreadsMetric.AddAxi("Refresh Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
 
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "refresh_threads",
-		Field: "payload.elasticsearch.node_stats.thread_pool.refresh.threads",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: refreshThreadsMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	refreshQueueMetric := newMetricItem("refresh_queue", 1, ThreadPoolRefreshGroupKey)
-	refreshQueueMetric.AddAxi("Refresh Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
 
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "refresh_queue",
-		Field: "payload.elasticsearch.node_stats.thread_pool.refresh.queue",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: refreshQueueMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	refreshActiveMetric := newMetricItem("refresh_active", 1, ThreadPoolRefreshGroupKey)
-	refreshActiveMetric.AddAxi("Refresh Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "refresh_active",
-		Field: "payload.elasticsearch.node_stats.thread_pool.refresh.active",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: refreshActiveMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	refreshRejectedMetric := newMetricItem("refresh_rejected", 1, ThreadPoolRefreshGroupKey)
-	refreshRejectedMetric.AddAxi("Refresh Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "refresh_rejected",
-		Field: "payload.elasticsearch.node_stats.thread_pool.refresh.rejected",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: refreshRejectedMetric,
-		FormatType: "num",
-		Units: "rejected/s",
-	})
-	forceMergeThreadsMetric := newMetricItem("force_merge_threads", 1, ThreadPoolForceMergeGroupKey)
-	forceMergeThreadsMetric.AddAxi("Force Merge Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "force_merge_threads",
-		Field: "payload.elasticsearch.node_stats.thread_pool.force_merge.threads",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: forceMergeThreadsMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	forceMergeQueueMetric := newMetricItem("force_merge_queue", 1, ThreadPoolForceMergeGroupKey)
-	forceMergeQueueMetric.AddAxi("Force Merge Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "force_merge_queue",
-		Field: "payload.elasticsearch.node_stats.thread_pool.force_merge.queue",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: forceMergeQueueMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	forceMergeActiveMetric := newMetricItem("force_merge_active", 1, ThreadPoolForceMergeGroupKey)
-	forceMergeActiveMetric.AddAxi("Force Merge Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "force_merge_active",
-		Field: "payload.elasticsearch.node_stats.thread_pool.force_merge.active",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: forceMergeActiveMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	forceMergeRejectedMetric := newMetricItem("force_merge_rejected", 1, ThreadPoolForceMergeGroupKey)
-	forceMergeRejectedMetric.AddAxi("Force Merge Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "force_merge_rejected",
-		Field: "payload.elasticsearch.node_stats.thread_pool.force_merge.rejected",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: forceMergeRejectedMetric,
-		FormatType: "num",
-		Units: "rejected/s",
-	})
 	//Get Thread Pool queue
 	aggs:=map[string]interface{}{}
 
@@ -575,5 +636,5 @@ func (h *APIHandler) getThreadPoolMetrics(clusterID string, bucketSize int, min,
 			},
 		},
 	}
-	return h.getMetrics(query, queueMetricItems, bucketSize), nil
+	return h.getMetrics(ctx, query, queueMetricItems, bucketSize), nil
 }

modules/elastic/api/v1/index_metrics.go

@@ -24,6 +24,7 @@
 package v1
 
 import (
+	"context"
 	"fmt"
 	log "github.com/cihub/seelog"
 	"infini.sh/framework/core/elastic"
@@ -37,7 +38,44 @@ import (
 	"time"
 )
 
-func (h *APIHandler) getIndexMetrics(req *http.Request, clusterID string, bucketSize int, min, max int64, indexName string, top int) map[string]*common.MetricItem{
+const (
+	IndexStorageMetricKey = "index_storage"
+	SegmentCountMetricKey = "segment_count"
+	DocCountMetricKey = "doc_count"
+	DocsDeletedMetricKey = "docs_deleted"
+	QueryTimesMetricKey = "query_times"
+	FetchTimesMetricKey = "fetch_times"
+	ScrollTimesMetricKey = "scroll_times"
+	MergeTimesMetricKey = "merge_times"
+	RefreshTimesMetricKey = "refresh_times"
+	FlushTimesMetricKey = "flush_times"
+	IndexingRateMetricKey = "indexing_rate"
+	IndexingBytesMetricKey = "indexing_bytes"
+	IndexingLatencyMetricKey = "indexing_latency"
+	QueryLatencyMetricKey = "query_latency"
+	FetchLatencyMetricKey = "fetch_latency"
+	MergeLatencyMetricKey = "merge_latency"
+	RefreshLatencyMetricKey = "refresh_latency"
+	ScrollLatencyMetricKey = "scroll_latency"
+	FlushLatencyMetricKey = "flush_latency"
+	QueryCacheMetricKey = "query_cache"
+	RequestCacheMetricKey = "request_cache"
+	RequestCacheHitMetricKey = "request_cache_hit"
+	RequestCacheMissMetricKey = "request_cache_miss"
+	QueryCacheCountMetricKey = "query_cache_count"
+	QueryCacheHitMetricKey = "query_cache_hit"
+	QueryCacheMissMetricKey = "query_cache_miss"
+	FielddataCacheMetricKey = "fielddata_cache"
+	SegmentMemoryMetricKey = "segment_memory"
+	SegmentDocValuesMemoryMetricKey = "segment_doc_values_memory"
+	SegmentTermsMemoryMetricKey = "segment_terms_memory"
+	SegmentFieldsMemoryMetricKey = "segment_fields_memory"
+	SegmentIndexWriterMemoryMetricKey = "segment_index_writer_memory"
+	SegmentTermVectorsMemoryMetricKey = "segment_term_vectors_memory"
+	DocPercentMetricKey = "doc_percent"
+)
+
+func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clusterID string, bucketSize int, min, max int64, indexName string, top int, metricKey string) map[string]*common.MetricItem{
 	bucketSizeStr:=fmt.Sprintf("%vs",bucketSize)
 
 	var must = []util.MapStr{
@@ -128,455 +166,470 @@ func (h *APIHandler) getIndexMetrics(req *http.Request, clusterID string, bucket
 			},
 		},
 	}
-	//索引存储大小
+	indexMetricItems := []GroupMetricItem{}
-	indexStorageMetric := newMetricItem("index_storage", 1, StorageGroupKey)
+	switch metricKey {
-	indexStorageMetric.AddAxi("Index storage","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
+	case IndexStorageMetricKey:
+		//索引存储大小
+		indexStorageMetric := newMetricItem(IndexStorageMetricKey, 1, StorageGroupKey)
+		indexStorageMetric.AddAxi("Index storage", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key:          "index_storage",
+			Field:        "payload.elasticsearch.index_stats.total.store.size_in_bytes",
+			ID:           util.GetUUID(),
+			IsDerivative: false,
+			MetricItem:   indexStorageMetric,
+			FormatType:   "bytes",
+			Units:        "",
+		})
+	case SegmentCountMetricKey:
+		// segment 数量
+		segmentCountMetric := newMetricItem(SegmentCountMetricKey, 15, StorageGroupKey)
+		segmentCountMetric.AddAxi("segment count", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key:          "segment_count",
+			Field:        "payload.elasticsearch.index_stats.total.segments.count",
+			ID:           util.GetUUID(),
+			IsDerivative: false,
+			MetricItem:   segmentCountMetric,
+			FormatType:   "num",
+			Units:        "",
+		})
+	case DocCountMetricKey:
+		//索引文档个数
+		docCountMetric := newMetricItem(DocCountMetricKey, 2, DocumentGroupKey)
+		docCountMetric.AddAxi("Doc count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
 
-	indexMetricItems := []GroupMetricItem{
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		{
+			Key:          "doc_count",
-			Key: "index_storage",
+			Field:        "payload.elasticsearch.index_stats.total.docs.count",
-			Field: "payload.elasticsearch.index_stats.total.store.size_in_bytes",
+			ID:           util.GetUUID(),
+			IsDerivative: false,
+			MetricItem:   docCountMetric,
+			FormatType:   "num",
+			Units:        "",
+		})
+	case DocsDeletedMetricKey:
+		// docs 删除数量
+		docsDeletedMetric := newMetricItem(DocsDeletedMetricKey, 17, DocumentGroupKey)
+		docsDeletedMetric.AddAxi("docs deleted", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key:          "docs_deleted",
+			Field:        "payload.elasticsearch.index_stats.total.docs.deleted",
+			ID:           util.GetUUID(),
+			IsDerivative: false,
+			MetricItem:   docsDeletedMetric,
+			FormatType:   "num",
+			Units:        "",
+		})
+	case QueryTimesMetricKey:
+		//查询次数
+		queryTimesMetric := newMetricItem(QueryTimesMetricKey, 2, OperationGroupKey)
+		queryTimesMetric.AddAxi("Query times", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
+
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key:          "query_times",
+			Field:        "payload.elasticsearch.index_stats.total.search.query_total",
+			ID:           util.GetUUID(),
+			IsDerivative: true,
+			MetricItem:   queryTimesMetric,
+			FormatType:   "num",
+			Units:        "requests/s",
+		})
+	case FetchTimesMetricKey:
+		//Fetch次数
+		fetchTimesMetric := newMetricItem(FetchTimesMetricKey, 3, OperationGroupKey)
+		fetchTimesMetric.AddAxi("Fetch times", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key:          "fetch_times",
+			Field:        "payload.elasticsearch.index_stats.total.search.fetch_total",
+			ID:           util.GetUUID(),
+			IsDerivative: true,
+			MetricItem:   fetchTimesMetric,
+			FormatType:   "num",
+			Units:        "requests/s",
+		})
+	case ScrollTimesMetricKey:
+		//scroll 次数
+		scrollTimesMetric := newMetricItem(ScrollTimesMetricKey, 4, OperationGroupKey)
+		scrollTimesMetric.AddAxi("scroll times", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key:          "scroll_times",
+			Field:        "payload.elasticsearch.index_stats.total.search.scroll_total",
+			ID:           util.GetUUID(),
+			IsDerivative: true,
+			MetricItem:   scrollTimesMetric,
+			FormatType:   "num",
+			Units:        "requests/s",
+		})
+	case MergeTimesMetricKey:
+		//Merge次数
+		mergeTimesMetric := newMetricItem(MergeTimesMetricKey, 7, OperationGroupKey)
+		mergeTimesMetric.AddAxi("Merge times", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key:          "merge_times",
+			Field:        "payload.elasticsearch.index_stats.total.merges.total",
+			ID:           util.GetUUID(),
+			IsDerivative: true,
+			MetricItem:   mergeTimesMetric,
+			FormatType:   "num",
+			Units:        "requests/s",
+		})
+	case RefreshTimesMetricKey:
+		//Refresh次数
+		refreshTimesMetric := newMetricItem(RefreshTimesMetricKey, 5, OperationGroupKey)
+		refreshTimesMetric.AddAxi("Refresh times", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key:          "refresh_times",
+			Field:        "payload.elasticsearch.index_stats.total.refresh.total",
+			ID:           util.GetUUID(),
+			IsDerivative: true,
+			MetricItem:   refreshTimesMetric,
+			FormatType:   "num",
+			Units:        "requests/s",
+		})
+	case FlushTimesMetricKey:
+		//flush 次数
+		flushTimesMetric := newMetricItem(FlushTimesMetricKey, 6, OperationGroupKey)
+		flushTimesMetric.AddAxi("flush times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "flush_times",
+			Field: "payload.elasticsearch.index_stats.total.flush.total",
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: flushTimesMetric,
+			FormatType: "num",
+			Units: "requests/s",
+		})
+	case IndexingRateMetricKey:
+		//写入速率
+		indexingRateMetric := newMetricItem(IndexingRateMetricKey, 1, OperationGroupKey)
+		indexingRateMetric.AddAxi("Indexing rate","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "indexing_rate",
+			Field: "payload.elasticsearch.index_stats.primaries.indexing.index_total",
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: indexingRateMetric,
+			FormatType: "num",
+			Units: "doc/s",
+		})
+	case IndexingBytesMetricKey:
+		indexingBytesMetric := newMetricItem(IndexingBytesMetricKey, 2, OperationGroupKey)
+		indexingBytesMetric.AddAxi("Indexing bytes","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key:          "indexing_bytes",
+			Field:        "payload.elasticsearch.index_stats.primaries.store.size_in_bytes",
+			ID:           util.GetUUID(),
+			IsDerivative: true,
+			MetricItem:   indexingBytesMetric,
+			FormatType:   "bytes",
+			Units:        "bytes/s",
+		})
+	case IndexingLatencyMetricKey:
+		//写入时延
+		indexingLatencyMetric := newMetricItem(IndexingLatencyMetricKey, 1, LatencyGroupKey)
+		indexingLatencyMetric.AddAxi("Indexing latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "indexing_latency",
+			Field: "payload.elasticsearch.index_stats.primaries.indexing.index_time_in_millis",
+			Field2: "payload.elasticsearch.index_stats.primaries.indexing.index_total",
+			Calc: func(value, value2 float64) float64 {
+				return value/value2
+			},
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: indexingLatencyMetric,
+			FormatType: "num",
+			Units: "ms",
+		})
+	case QueryLatencyMetricKey:
+		//查询时延
+		queryLatencyMetric := newMetricItem(QueryLatencyMetricKey, 2, LatencyGroupKey)
+		queryLatencyMetric.AddAxi("Query latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "query_latency",
+			Field: "payload.elasticsearch.index_stats.total.search.query_time_in_millis",
+			Field2: "payload.elasticsearch.index_stats.total.search.query_total",
+			Calc: func(value, value2 float64) float64 {
+				return value/value2
+			},
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: queryLatencyMetric,
+			FormatType: "num",
+			Units: "ms",
+		})
+	case FetchLatencyMetricKey:
+		//fetch时延
+		fetchLatencyMetric := newMetricItem(FetchLatencyMetricKey, 3, LatencyGroupKey)
+		fetchLatencyMetric.AddAxi("Fetch latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "fetch_latency",
+			Field: "payload.elasticsearch.index_stats.total.search.fetch_time_in_millis",
+			Field2: "payload.elasticsearch.index_stats.total.search.fetch_total",
+			Calc: func(value, value2 float64) float64 {
+				return value/value2
+			},
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: fetchLatencyMetric,
+			FormatType: "num",
+			Units: "ms",
+		})
+	case MergeLatencyMetricKey:
+		//merge时延
+		mergeLatencyMetric := newMetricItem(MergeLatencyMetricKey, 7, LatencyGroupKey)
+		mergeLatencyMetric.AddAxi("Merge latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "merge_latency",
+			Field: "payload.elasticsearch.index_stats.total.merges.total_time_in_millis",
+			Field2: "payload.elasticsearch.index_stats.total.merges.total",
+			Calc: func(value, value2 float64) float64 {
+				return value/value2
+			},
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: mergeLatencyMetric,
+			FormatType: "num",
+			Units: "ms",
+		})
+	case RefreshLatencyMetricKey:
+
+		//refresh时延
+		refreshLatencyMetric := newMetricItem(RefreshLatencyMetricKey, 5, LatencyGroupKey)
+		refreshLatencyMetric.AddAxi("Refresh latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "refresh_latency",
+			Field: "payload.elasticsearch.index_stats.total.refresh.total_time_in_millis",
+			Field2: "payload.elasticsearch.index_stats.total.refresh.total",
+			Calc: func(value, value2 float64) float64 {
+				return value/value2
+			},
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: refreshLatencyMetric,
+			FormatType: "num",
+			Units: "ms",
+		})
+	case ScrollLatencyMetricKey:
+		//scroll时延
+		scrollLatencyMetric := newMetricItem(ScrollLatencyMetricKey, 4, LatencyGroupKey)
+		scrollLatencyMetric.AddAxi("Scroll Latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "scroll_latency",
+			Field: "payload.elasticsearch.index_stats.total.search.scroll_time_in_millis",
+			Field2: "payload.elasticsearch.index_stats.total.search.scroll_total",
+			Calc: func(value, value2 float64) float64 {
+				return value/value2
+			},
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: scrollLatencyMetric,
+			FormatType: "num",
+			Units: "ms",
+		})
+	case FlushLatencyMetricKey:
+		//flush 时延
+		flushLatencyMetric := newMetricItem(FlushLatencyMetricKey, 6, LatencyGroupKey)
+		flushLatencyMetric.AddAxi("Flush latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "flush_latency",
+			Field: "payload.elasticsearch.index_stats.total.flush.total_time_in_millis",
+			Field2: "payload.elasticsearch.index_stats.total.flush.total",
+			Calc: func(value, value2 float64) float64 {
+				return value/value2
+			},
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: flushLatencyMetric,
+			FormatType: "num",
+			Units: "ms",
+		})
+	case QueryCacheMetricKey:
+		//queryCache
+		queryCacheMetric := newMetricItem(QueryCacheMetricKey, 1, CacheGroupKey)
+		queryCacheMetric.AddAxi("Query cache","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "query_cache",
+			Field: "payload.elasticsearch.index_stats.total.query_cache.memory_size_in_bytes",
 			ID: util.GetUUID(),
 			IsDerivative: false,
-			MetricItem: indexStorageMetric,
+			MetricItem: queryCacheMetric,
 			FormatType: "bytes",
 			Units: "",
-		},
+		})
+	case RequestCacheMetricKey:
+		//requestCache
+		requestCacheMetric := newMetricItem(RequestCacheMetricKey, 2, CacheGroupKey)
+		requestCacheMetric.AddAxi("request cache","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "request_cache",
+			Field: "payload.elasticsearch.index_stats.total.request_cache.memory_size_in_bytes",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: requestCacheMetric,
+			FormatType: "bytes",
+			Units: "",
+		})
+	case RequestCacheHitMetricKey:
+		// Request Cache Hit
+		requestCacheHitMetric:=newMetricItem(RequestCacheHitMetricKey, 6, CacheGroupKey)
+		requestCacheHitMetric.AddAxi("request cache hit","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
+		indexMetricItems=append(indexMetricItems, GroupMetricItem{
+			Key: "request_cache_hit",
+			Field: "payload.elasticsearch.index_stats.total.request_cache.hit_count",
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: requestCacheHitMetric,
+			FormatType: "num",
+			Units: "hits",
+		})
+	case RequestCacheMissMetricKey:
+		// Request Cache Miss
+		requestCacheMissMetric:=newMetricItem(RequestCacheMissMetricKey, 8, CacheGroupKey)
+		requestCacheMissMetric.AddAxi("request cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
+		indexMetricItems=append(indexMetricItems, GroupMetricItem{
+			Key: "request_cache_miss",
+			Field: "payload.elasticsearch.index_stats.total.request_cache.miss_count",
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: requestCacheMissMetric,
+			FormatType: "num",
+			Units: "misses",
+		})
+	case QueryCacheCountMetricKey:
+		// Query Cache Count
+		queryCacheCountMetric:=newMetricItem(QueryCacheCountMetricKey, 4, CacheGroupKey)
+		queryCacheCountMetric.AddAxi("query cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
+		indexMetricItems=append(indexMetricItems, GroupMetricItem{
+			Key: "query_cache_count",
+			Field: "payload.elasticsearch.index_stats.total.query_cache.cache_count",
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: queryCacheCountMetric,
+			FormatType: "num",
+			Units: "",
+		})
+	case QueryCacheHitMetricKey:
+		// Query Cache Miss
+		queryCacheHitMetric:=newMetricItem(QueryCacheHitMetricKey, 5, CacheGroupKey)
+		queryCacheHitMetric.AddAxi("query cache hit","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
+		indexMetricItems=append(indexMetricItems, GroupMetricItem{
+			Key: "query_cache_hit",
+			Field: "payload.elasticsearch.index_stats.total.query_cache.hit_count",
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: queryCacheHitMetric,
+			FormatType: "num",
+			Units: "hits",
+		})
+	case QueryCacheMissMetricKey:
+		// Query Cache Miss
+		queryCacheMissMetric:=newMetricItem(QueryCacheMissMetricKey, 7, CacheGroupKey)
+		queryCacheMissMetric.AddAxi("query cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
+		indexMetricItems=append(indexMetricItems, GroupMetricItem{
+			Key: "query_cache_miss",
+			Field: "payload.elasticsearch.index_stats.total.query_cache.miss_count",
+			ID: util.GetUUID(),
+			IsDerivative: true,
+			MetricItem: queryCacheMissMetric,
+			FormatType: "num",
+			Units: "misses",
+		})
+	case FielddataCacheMetricKey:
+		// Fielddata内存占用大小
+		fieldDataCacheMetric:=newMetricItem(FielddataCacheMetricKey, 3, CacheGroupKey)
+		fieldDataCacheMetric.AddAxi("FieldData Cache","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true)
+		indexMetricItems=append(indexMetricItems, GroupMetricItem{
+			Key: "fielddata_cache",
+			Field: "payload.elasticsearch.index_stats.total.fielddata.memory_size_in_bytes",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: fieldDataCacheMetric,
+			FormatType: "bytes",
+			Units: "",
+		})
+	case SegmentMemoryMetricKey:
+		//segment memory
+		segmentMemoryMetric := newMetricItem(SegmentMemoryMetricKey, 13, MemoryGroupKey)
+		segmentMemoryMetric.AddAxi("Segment memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "segment_memory",
+			Field: "payload.elasticsearch.index_stats.total.segments.memory_in_bytes",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: segmentMemoryMetric,
+			FormatType: "bytes",
+			Units: "",
+		})
+	case SegmentDocValuesMemoryMetricKey:
+		//segment doc values memory
+		docValuesMemoryMetric := newMetricItem(SegmentDocValuesMemoryMetricKey, 13, MemoryGroupKey)
+		docValuesMemoryMetric.AddAxi("Segment Doc values Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "segment_doc_values_memory",
+			Field: "payload.elasticsearch.index_stats.total.segments.doc_values_memory_in_bytes",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: docValuesMemoryMetric,
+			FormatType: "bytes",
+			Units: "",
+		})
+	case SegmentTermsMemoryMetricKey:
+		//segment terms memory
+		termsMemoryMetric := newMetricItem(SegmentTermsMemoryMetricKey, 13, MemoryGroupKey)
+		termsMemoryMetric.AddAxi("Segment Terms Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "segment_terms_memory",
+			Field: "payload.elasticsearch.index_stats.total.segments.terms_memory_in_bytes",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: termsMemoryMetric,
+			FormatType: "bytes",
+			Units: "",
+		})
+	case SegmentFieldsMemoryMetricKey:
+		//segment fields memory
+		fieldsMemoryMetric := newMetricItem(SegmentFieldsMemoryMetricKey, 13, MemoryGroupKey)
+		fieldsMemoryMetric.AddAxi("Segment Fields Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
+		indexMetricItems = append(indexMetricItems, GroupMetricItem{
+			Key: "segment_fields_memory",
+			Field: "payload.elasticsearch.index_stats.total.segments.stored_fields_memory_in_bytes",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: fieldsMemoryMetric,
+			FormatType: "bytes",
+			Units: "",
+		})
+	case SegmentIndexWriterMemoryMetricKey:
+		// segment index writer memory
+		segmentIndexWriterMemoryMetric:=newMetricItem(SegmentIndexWriterMemoryMetricKey, 16, MemoryGroupKey)
+		segmentIndexWriterMemoryMetric.AddAxi("segment doc values memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true)
+		indexMetricItems=append(indexMetricItems, GroupMetricItem{
+			Key: "segment_index_writer_memory",
+			Field: "payload.elasticsearch.index_stats.total.segments.index_writer_memory_in_bytes",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: segmentIndexWriterMemoryMetric,
+			FormatType: "bytes",
+			Units: "",
+		})
+	case SegmentTermVectorsMemoryMetricKey:
+		// segment term vectors memory
+		segmentTermVectorsMemoryMetric:=newMetricItem(SegmentTermVectorsMemoryMetricKey, 16, MemoryGroupKey)
+		segmentTermVectorsMemoryMetric.AddAxi("segment term vectors memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true)
+		indexMetricItems=append(indexMetricItems, GroupMetricItem{
+			Key: "segment_term_vectors_memory",
+			Field: "payload.elasticsearch.index_stats.total.segments.term_vectors_memory_in_bytes",
+			ID: util.GetUUID(),
+			IsDerivative: false,
+			MetricItem: segmentTermVectorsMemoryMetric,
+			FormatType: "bytes",
+			Units: "",
+		})
+
 	}
-	// segment 数量
-	segmentCountMetric:=newMetricItem("segment_count", 15, StorageGroupKey)
-	segmentCountMetric.AddAxi("segment count","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
-	indexMetricItems=append(indexMetricItems, GroupMetricItem{
-		Key: "segment_count",
-		Field: "payload.elasticsearch.index_stats.total.segments.count",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: segmentCountMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	//索引文档个数
-	docCountMetric := newMetricItem("doc_count", 2, DocumentGroupKey)
-	docCountMetric.AddAxi("Doc count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
 
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "doc_count",
-		Field: "payload.elasticsearch.index_stats.total.docs.count",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: docCountMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	// docs 删除数量
-	docsDeletedMetric:=newMetricItem("docs_deleted", 17, DocumentGroupKey)
-	docsDeletedMetric.AddAxi("docs deleted","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
-	indexMetricItems=append(indexMetricItems, GroupMetricItem{
-		Key: "docs_deleted",
-		Field: "payload.elasticsearch.index_stats.total.docs.deleted",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: docsDeletedMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	//查询次数
-	queryTimesMetric := newMetricItem("query_times", 2, OperationGroupKey)
-	queryTimesMetric.AddAxi("Query times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "query_times",
-		Field: "payload.elasticsearch.index_stats.total.search.query_total",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: queryTimesMetric,
-		FormatType: "num",
-		Units: "requests/s",
-	})
-
-	//Fetch次数
-	fetchTimesMetric := newMetricItem("fetch_times", 3, OperationGroupKey)
-	fetchTimesMetric.AddAxi("Fetch times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "fetch_times",
-		Field: "payload.elasticsearch.index_stats.total.search.fetch_total",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: fetchTimesMetric,
-		FormatType: "num",
-		Units: "requests/s",
-	})
-	//scroll 次数
-	scrollTimesMetric := newMetricItem("scroll_times", 4, OperationGroupKey)
-	scrollTimesMetric.AddAxi("scroll times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "scroll_times",
-		Field: "payload.elasticsearch.index_stats.total.search.scroll_total",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: scrollTimesMetric,
-		FormatType: "num",
-		Units: "requests/s",
-	})
-	//Merge次数
-	mergeTimesMetric := newMetricItem("merge_times", 7, OperationGroupKey)
-	mergeTimesMetric.AddAxi("Merge times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "merge_times",
-		Field: "payload.elasticsearch.index_stats.total.merges.total",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: mergeTimesMetric,
-		FormatType: "num",
-		Units: "requests/s",
-	})
-	//Refresh次数
-	refreshTimesMetric := newMetricItem("refresh_times", 5, OperationGroupKey)
-	refreshTimesMetric.AddAxi("Refresh times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "refresh_times",
-		Field: "payload.elasticsearch.index_stats.total.refresh.total",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: refreshTimesMetric,
-		FormatType: "num",
-		Units: "requests/s",
-	})
-	//flush 次数
-	flushTimesMetric := newMetricItem("flush_times", 6, OperationGroupKey)
-	flushTimesMetric.AddAxi("flush times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "flush_times",
-		Field: "payload.elasticsearch.index_stats.total.flush.total",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: flushTimesMetric,
-		FormatType: "num",
-		Units: "requests/s",
-	})
-
-	//写入速率
-	indexingRateMetric := newMetricItem("indexing_rate", 1, OperationGroupKey)
-	indexingRateMetric.AddAxi("Indexing rate","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "indexing_rate",
-		Field: "payload.elasticsearch.index_stats.primaries.indexing.index_total",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: indexingRateMetric,
-		FormatType: "num",
-		Units: "doc/s",
-	})
-	indexingBytesMetric := newMetricItem("indexing_bytes", 2, OperationGroupKey)
-	indexingBytesMetric.AddAxi("Indexing bytes","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key:          "indexing_bytes",
-		Field:        "payload.elasticsearch.index_stats.primaries.store.size_in_bytes",
-		ID:           util.GetUUID(),
-		IsDerivative: true,
-		MetricItem:   indexingBytesMetric,
-		FormatType:   "bytes",
-		Units:        "bytes/s",
-	})
-	//写入时延
-	indexingLatencyMetric := newMetricItem("indexing_latency", 1, LatencyGroupKey)
-	indexingLatencyMetric.AddAxi("Indexing latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "indexing_latency",
-		Field: "payload.elasticsearch.index_stats.primaries.indexing.index_time_in_millis",
-		Field2: "payload.elasticsearch.index_stats.primaries.indexing.index_total",
-		Calc: func(value, value2 float64) float64 {
-			return value/value2
-		},
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: indexingLatencyMetric,
-		FormatType: "num",
-		Units: "ms",
-	})
-
-	//查询时延
-	queryLatencyMetric := newMetricItem("query_latency", 2, LatencyGroupKey)
-	queryLatencyMetric.AddAxi("Query latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "query_latency",
-		Field: "payload.elasticsearch.index_stats.total.search.query_time_in_millis",
-		Field2: "payload.elasticsearch.index_stats.total.search.query_total",
-		Calc: func(value, value2 float64) float64 {
-			return value/value2
-		},
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: queryLatencyMetric,
-		FormatType: "num",
-		Units: "ms",
-	})
-	//fetch时延
-	fetchLatencyMetric := newMetricItem("fetch_latency", 3, LatencyGroupKey)
-	fetchLatencyMetric.AddAxi("Fetch latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "fetch_latency",
-		Field: "payload.elasticsearch.index_stats.total.search.fetch_time_in_millis",
-		Field2: "payload.elasticsearch.index_stats.total.search.fetch_total",
-		Calc: func(value, value2 float64) float64 {
-			return value/value2
-		},
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: fetchLatencyMetric,
-		FormatType: "num",
-		Units: "ms",
-	})
-
-	//merge时延
-	mergeLatencyMetric := newMetricItem("merge_latency", 7, LatencyGroupKey)
-	mergeLatencyMetric.AddAxi("Merge latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "merge_latency",
-		Field: "payload.elasticsearch.index_stats.total.merges.total_time_in_millis",
-		Field2: "payload.elasticsearch.index_stats.total.merges.total",
-		Calc: func(value, value2 float64) float64 {
-			return value/value2
-		},
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: mergeLatencyMetric,
-		FormatType: "num",
-		Units: "ms",
-	})
-	//refresh时延
-	refreshLatencyMetric := newMetricItem("refresh_latency", 5, LatencyGroupKey)
-	refreshLatencyMetric.AddAxi("Refresh latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "refresh_latency",
-		Field: "payload.elasticsearch.index_stats.total.refresh.total_time_in_millis",
-		Field2: "payload.elasticsearch.index_stats.total.refresh.total",
-		Calc: func(value, value2 float64) float64 {
-			return value/value2
-		},
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: refreshLatencyMetric,
-		FormatType: "num",
-		Units: "ms",
-	})
-	//scroll时延
-	scrollLatencyMetric := newMetricItem("scroll_latency", 4, LatencyGroupKey)
-	scrollLatencyMetric.AddAxi("Scroll Latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "scroll_latency",
-		Field: "payload.elasticsearch.index_stats.total.search.scroll_time_in_millis",
-		Field2: "payload.elasticsearch.index_stats.total.search.scroll_total",
-		Calc: func(value, value2 float64) float64 {
-			return value/value2
-		},
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: scrollLatencyMetric,
-		FormatType: "num",
-		Units: "ms",
-	})
-	//flush 时延
-	flushLatencyMetric := newMetricItem("flush_latency", 6, LatencyGroupKey)
-	flushLatencyMetric.AddAxi("Flush latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "flush_latency",
-		Field: "payload.elasticsearch.index_stats.total.flush.total_time_in_millis",
-		Field2: "payload.elasticsearch.index_stats.total.flush.total",
-		Calc: func(value, value2 float64) float64 {
-			return value/value2
-		},
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: flushLatencyMetric,
-		FormatType: "num",
-		Units: "ms",
-	})
-	//queryCache
-	queryCacheMetric := newMetricItem("query_cache", 1, CacheGroupKey)
-	queryCacheMetric.AddAxi("Query cache","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "query_cache",
-		Field: "payload.elasticsearch.index_stats.total.query_cache.memory_size_in_bytes",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: queryCacheMetric,
-		FormatType: "bytes",
-		Units: "",
-	})
-	//requestCache
-	requestCacheMetric := newMetricItem("request_cache", 2, CacheGroupKey)
-	requestCacheMetric.AddAxi("request cache","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "request_cache",
-		Field: "payload.elasticsearch.index_stats.total.request_cache.memory_size_in_bytes",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: requestCacheMetric,
-		FormatType: "bytes",
-		Units: "",
-	})
-	// Request Cache Hit
-	requestCacheHitMetric:=newMetricItem("request_cache_hit", 6, CacheGroupKey)
-	requestCacheHitMetric.AddAxi("request cache hit","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
-	indexMetricItems=append(indexMetricItems, GroupMetricItem{
-		Key: "request_cache_hit",
-		Field: "payload.elasticsearch.index_stats.total.request_cache.hit_count",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: requestCacheHitMetric,
-		FormatType: "num",
-		Units: "hits",
-	})
-	// Request Cache Miss
-	requestCacheMissMetric:=newMetricItem("request_cache_miss", 8, CacheGroupKey)
-	requestCacheMissMetric.AddAxi("request cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
-	indexMetricItems=append(indexMetricItems, GroupMetricItem{
-		Key: "request_cache_miss",
-		Field: "payload.elasticsearch.index_stats.total.request_cache.miss_count",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: requestCacheMissMetric,
-		FormatType: "num",
-		Units: "misses",
-	})
-	// Query Cache Count
-	queryCacheCountMetric:=newMetricItem("query_cache_count", 4, CacheGroupKey)
-	queryCacheCountMetric.AddAxi("query cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
-	indexMetricItems=append(indexMetricItems, GroupMetricItem{
-		Key: "query_cache_count",
-		Field: "payload.elasticsearch.index_stats.total.query_cache.cache_count",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: queryCacheCountMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	// Query Cache Miss
-	queryCacheHitMetric:=newMetricItem("query_cache_hit", 5, CacheGroupKey)
-	queryCacheHitMetric.AddAxi("query cache hit","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
-	indexMetricItems=append(indexMetricItems, GroupMetricItem{
-		Key: "query_cache_hit",
-		Field: "payload.elasticsearch.index_stats.total.query_cache.hit_count",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: queryCacheHitMetric,
-		FormatType: "num",
-		Units: "hits",
-	})
-
-	//// Query Cache evictions
-	//queryCacheEvictionsMetric:=newMetricItem("query_cache_evictions", 11, CacheGroupKey)
-	//queryCacheEvictionsMetric.AddAxi("query cache evictions","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
-	//indexMetricItems=append(indexMetricItems, GroupMetricItem{
-	//	Key: "query_cache_evictions",
-	//	Field: "payload.elasticsearch.index_stats.total.query_cache.evictions",
-	//	ID: util.GetUUID(),
-	//	IsDerivative: true,
-	//	MetricItem: queryCacheEvictionsMetric,
-	//	FormatType: "num",
-	//	Units: "evictions",
-	//})
-
-	// Query Cache Miss
-	queryCacheMissMetric:=newMetricItem("query_cache_miss", 7, CacheGroupKey)
-	queryCacheMissMetric.AddAxi("query cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
-	indexMetricItems=append(indexMetricItems, GroupMetricItem{
-		Key: "query_cache_miss",
-		Field: "payload.elasticsearch.index_stats.total.query_cache.miss_count",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: queryCacheMissMetric,
-		FormatType: "num",
-		Units: "misses",
-	})
-	// Fielddata内存占用大小
-	fieldDataCacheMetric:=newMetricItem("fielddata_cache", 3, CacheGroupKey)
-	fieldDataCacheMetric.AddAxi("FieldData Cache","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true)
-	indexMetricItems=append(indexMetricItems, GroupMetricItem{
-		Key: "fielddata_cache",
-		Field: "payload.elasticsearch.index_stats.total.fielddata.memory_size_in_bytes",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: fieldDataCacheMetric,
-		FormatType: "bytes",
-		Units: "",
-	})
-	//segment memory
-	segmentMemoryMetric := newMetricItem("segment_memory", 13, MemoryGroupKey)
-	segmentMemoryMetric.AddAxi("Segment memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "segment_memory",
-		Field: "payload.elasticsearch.index_stats.total.segments.memory_in_bytes",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: segmentMemoryMetric,
-		FormatType: "bytes",
-		Units: "",
-	})
-
-	//segment doc values memory
-	docValuesMemoryMetric := newMetricItem("segment_doc_values_memory", 13, MemoryGroupKey)
-	docValuesMemoryMetric.AddAxi("Segment Doc values Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "segment_doc_values_memory",
-		Field: "payload.elasticsearch.index_stats.total.segments.doc_values_memory_in_bytes",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: docValuesMemoryMetric,
-		FormatType: "bytes",
-		Units: "",
-	})
-
-	//segment terms memory
-	termsMemoryMetric := newMetricItem("segment_terms_memory", 13, MemoryGroupKey)
-	termsMemoryMetric.AddAxi("Segment Terms Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "segment_terms_memory",
-		Field: "payload.elasticsearch.index_stats.total.segments.terms_memory_in_bytes",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: termsMemoryMetric,
-		FormatType: "bytes",
-		Units: "",
-	})
-
-	//segment fields memory
-	fieldsMemoryMetric := newMetricItem("segment_fields_memory", 13, MemoryGroupKey)
-	fieldsMemoryMetric.AddAxi("Segment Fields Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true)
-	indexMetricItems = append(indexMetricItems, GroupMetricItem{
-		Key: "segment_fields_memory",
-		Field: "payload.elasticsearch.index_stats.total.segments.stored_fields_memory_in_bytes",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: fieldsMemoryMetric,
-		FormatType: "bytes",
-		Units: "",
-	})
-	// segment index writer memory
-	segmentIndexWriterMemoryMetric:=newMetricItem("segment_index_writer_memory", 16, MemoryGroupKey)
-	segmentIndexWriterMemoryMetric.AddAxi("segment doc values memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true)
-	indexMetricItems=append(indexMetricItems, GroupMetricItem{
-		Key: "segment_index_writer_memory",
-		Field: "payload.elasticsearch.index_stats.total.segments.index_writer_memory_in_bytes",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: segmentIndexWriterMemoryMetric,
-		FormatType: "bytes",
-		Units: "",
-	})
-	// segment term vectors memory
-	segmentTermVectorsMemoryMetric:=newMetricItem("segment_term_vectors_memory", 16, MemoryGroupKey)
-	segmentTermVectorsMemoryMetric.AddAxi("segment term vectors memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true)
-	indexMetricItems=append(indexMetricItems, GroupMetricItem{
-		Key: "segment_term_vectors_memory",
-		Field: "payload.elasticsearch.index_stats.total.segments.term_vectors_memory_in_bytes",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: segmentTermVectorsMemoryMetric,
-		FormatType: "bytes",
-		Units: "",
-	})
 
 	aggs:=map[string]interface{}{}
 
@@ -642,7 +695,7 @@ func (h *APIHandler) getIndexMetrics(req *http.Request, clusterID string, bucket
 			},
 		},
 	}
-	return h.getMetrics(query, indexMetricItems, bucketSize)
+	return h.getMetrics(ctx, query, indexMetricItems, bucketSize)
 
 }
 

modules/elastic/api/v1/index_overview.go

@@ -28,6 +28,7 @@
 package v1
 
 import (
+	"context"
 	"fmt"
 	log "github.com/cihub/seelog"
 	httprouter "infini.sh/framework/core/api/router"
@@ -440,7 +441,7 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter,  req *http.Request, p
 			},
 		},
 	}
-	metrics := h.getMetrics(query, nodeMetricItems, bucketSize)
+	metrics := h.getMetrics(context.Background(), query, nodeMetricItems, bucketSize)
 	indexMetrics := map[string]util.MapStr{}
 	for key, item := range metrics {
 		for _, line := range item.Lines {
@@ -626,6 +627,8 @@ func (h *APIHandler) GetIndexShards(w http.ResponseWriter, req *http.Request, ps
 	h.WriteJSON(w, shardInfo, http.StatusOK)
 }
 
+const IndexHealthMetricKey = "index_health"
+
 func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
 	clusterID := ps.MustGetParameter("id")
 	indexName := ps.MustGetParameter("index")
@@ -699,63 +702,81 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ
 
 	bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
 	metricItems := []*common.MetricItem{}
-	metricItem:=newMetricItem("index_throughput", 1, OperationGroupKey)
+	metricKey := h.GetParameter(req, "key")
-	metricItem.AddAxi("indexing","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
+	timeout := h.GetParameterOrDefault(req, "timeout", "60s")
-	metricItem.AddLine("Indexing Rate","Primary Indexing","Number of documents being indexed for node.","group1","payload.elasticsearch.index_stats.primaries.indexing.index_total","max",bucketSizeStr,"doc/s","num","0,0.[00]","0,0.[00]",false,true)
+	du, err := time.ParseDuration(timeout)
-	metricItem.AddLine("Deleting Rate","Primary Deleting","Number of documents being deleted for node.","group1","payload.elasticsearch.index_stats.primaries.indexing.delete_total","max",bucketSizeStr,"doc/s","num","0,0.[00]","0,0.[00]",false,true)
-	metricItems=append(metricItems,metricItem)
-	metricItem=newMetricItem("search_throughput", 2, OperationGroupKey)
-	metricItem.AddAxi("searching","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,false)
-	metricItem.AddLine("Search Rate","Search Rate",
-		"Number of search requests being executed.",
-		"group1","payload.elasticsearch.index_stats.total.search.query_total","max",bucketSizeStr,"query/s","num","0,0.[00]","0,0.[00]",false,true)
-	metricItems=append(metricItems,metricItem)
-
-	metricItem=newMetricItem("index_latency", 3, LatencyGroupKey)
-	metricItem.AddAxi("indexing","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true)
-
-	metricItem.AddLine("Indexing Latency","Primary Indexing Latency","Average latency for indexing documents.","group1","payload.elasticsearch.index_stats.primaries.indexing.index_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true)
-	metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.index_stats.primaries.indexing.index_total"
-	metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
-		return value/value2
-	}
-	metricItem.AddLine("Deleting Latency","Primary Deleting Latency","Average latency for delete documents.","group1","payload.elasticsearch.index_stats.primaries.indexing.delete_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true)
-	metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.index_stats.primaries.indexing.delete_total"
-	metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
-		return value/value2
-	}
-	metricItems=append(metricItems,metricItem)
-
-	metricItem=newMetricItem("search_latency", 4, LatencyGroupKey)
-	metricItem.AddAxi("searching","group2",common.PositionLeft,"num","0,0","0,0.[00]",5,false)
-
-	metricItem.AddLine("Searching","Query Latency","Average latency for searching query.","group2","payload.elasticsearch.index_stats.total.search.query_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true)
-	metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.index_stats.total.search.query_total"
-	metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
-		return value/value2
-	}
-	metricItem.AddLine("Searching","Fetch Latency","Average latency for searching fetch.","group2","payload.elasticsearch.index_stats.total.search.fetch_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true)
-	metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.index_stats.total.search.fetch_total"
-	metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
-		return value/value2
-	}
-	metricItem.AddLine("Searching","Scroll Latency","Average latency for searching fetch.","group2","payload.elasticsearch.index_stats.total.search.scroll_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true)
-	metricItem.Lines[2].Metric.Field2 = "payload.elasticsearch.index_stats.total.search.scroll_total"
-	metricItem.Lines[2].Metric.Calc = func(value, value2 float64) float64 {
-		return value/value2
-	}
-	metricItems=append(metricItems,metricItem)
-	metrics := h.getSingleMetrics(metricItems,query, bucketSize)
-	healthMetric, err := h.getIndexHealthMetric(clusterID, indexName, min, max, bucketSize)
 	if err != nil {
 		log.Error(err)
+		h.WriteError(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), du)
+	defer cancel()
+	metrics := map[string]*common.MetricItem{}
+	if metricKey == IndexHealthMetricKey {
+		healthMetric, err := h.getIndexHealthMetric(ctx, clusterID, indexName, min, max, bucketSize)
+		if err != nil {
+			log.Error(err)
+		}
+		metrics["index_health"] = healthMetric
+	}else {
+		switch metricKey {
+		case IndexThroughputMetricKey:
+			metricItem := newMetricItem("index_throughput", 1, OperationGroupKey)
+			metricItem.AddAxi("indexing", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
+			metricItem.AddLine("Indexing Rate", "Primary Indexing", "Number of documents being indexed for node.", "group1", "payload.elasticsearch.index_stats.primaries.indexing.index_total", "max", bucketSizeStr, "doc/s", "num", "0,0.[00]", "0,0.[00]", false, true)
+			metricItem.AddLine("Deleting Rate", "Primary Deleting", "Number of documents being deleted for node.", "group1", "payload.elasticsearch.index_stats.primaries.indexing.delete_total", "max", bucketSizeStr, "doc/s", "num", "0,0.[00]", "0,0.[00]", false, true)
+			metricItems = append(metricItems, metricItem)
+		case SearchThroughputMetricKey:
+			metricItem := newMetricItem("search_throughput", 2, OperationGroupKey)
+			metricItem.AddAxi("searching", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
+			metricItem.AddLine("Search Rate", "Search Rate",
+				"Number of search requests being executed.",
+				"group1", "payload.elasticsearch.index_stats.total.search.query_total", "max", bucketSizeStr, "query/s", "num", "0,0.[00]", "0,0.[00]", false, true)
+			metricItems = append(metricItems, metricItem)
+		case IndexLatencyMetricKey:
+			metricItem := newMetricItem("index_latency", 3, LatencyGroupKey)
+			metricItem.AddAxi("indexing", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
+
+			metricItem.AddLine("Indexing Latency", "Primary Indexing Latency", "Average latency for indexing documents.", "group1", "payload.elasticsearch.index_stats.primaries.indexing.index_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
+			metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.index_stats.primaries.indexing.index_total"
+			metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
+				return value / value2
+			}
+			metricItem.AddLine("Deleting Latency", "Primary Deleting Latency", "Average latency for delete documents.", "group1", "payload.elasticsearch.index_stats.primaries.indexing.delete_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
+			metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.index_stats.primaries.indexing.delete_total"
+			metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
+				return value / value2
+			}
+			metricItems = append(metricItems, metricItem)
+		case SearchLatencyMetricKey:
+			metricItem := newMetricItem("search_latency", 4, LatencyGroupKey)
+			metricItem.AddAxi("searching", "group2", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
+
+			metricItem.AddLine("Searching", "Query Latency", "Average latency for searching query.", "group2", "payload.elasticsearch.index_stats.total.search.query_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
+			metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.index_stats.total.search.query_total"
+			metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
+				return value / value2
+			}
+			metricItem.AddLine("Searching", "Fetch Latency", "Average latency for searching fetch.", "group2", "payload.elasticsearch.index_stats.total.search.fetch_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
+			metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.index_stats.total.search.fetch_total"
+			metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
+				return value / value2
+			}
+			metricItem.AddLine("Searching", "Scroll Latency", "Average latency for searching fetch.", "group2", "payload.elasticsearch.index_stats.total.search.scroll_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
+			metricItem.Lines[2].Metric.Field2 = "payload.elasticsearch.index_stats.total.search.scroll_total"
+			metricItem.Lines[2].Metric.Calc = func(value, value2 float64) float64 {
+				return value / value2
+			}
+			metricItems = append(metricItems, metricItem)
+		}
+		metrics = h.getSingleMetrics(ctx, metricItems, query, bucketSize)
 	}
-	metrics["index_health"] = healthMetric
 	resBody["metrics"] = metrics
 	h.WriteJSON(w, resBody, http.StatusOK)
 }
 
-func (h *APIHandler) getIndexHealthMetric(id, indexName string, min, max int64, bucketSize int)(*common.MetricItem, error){
+func (h *APIHandler) getIndexHealthMetric(ctx context.Context, id, indexName string, min, max int64, bucketSize int)(*common.MetricItem, error){
 	bucketSizeStr:=fmt.Sprintf("%vs",bucketSize)
 	intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr)
 	if err != nil {
@@ -823,7 +844,8 @@ func (h *APIHandler) getIndexHealthMetric(id, indexName string, min, max int64,
 			},
 		},
 	}
-	response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query))
+	queryDSL := util.MustToJSONBytes(query)
+	response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).QueryDSL(ctx, getAllMetricsIndex(), nil, queryDSL)
 	if err != nil {
 		log.Error(err)
 		return nil, err
@@ -841,6 +863,7 @@ func (h *APIHandler) getIndexHealthMetric(id, indexName string, min, max int64,
 	}
 	metricItem.Lines[0].Data = metricData
 	metricItem.Lines[0].Type = common.GraphTypeBar
+	metricItem.Request = string(queryDSL)
 	return metricItem, nil
 }
 

modules/elastic/api/v1/manage.go

@@ -500,6 +500,7 @@ func (h *APIHandler) HandleMetricsSummaryAction(w http.ResponseWriter, req *http
 func (h *APIHandler) HandleClusterMetricsAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
 	resBody := map[string]interface{}{}
 	id := ps.ByName("id")
+	key := h.GetParameter(req, "key")
 
 	bucketSize, min, max, err := h.getMetricRangeAndBucketSize(req, 10, 90)
 	if err != nil {
@@ -514,13 +515,18 @@ func (h *APIHandler) HandleClusterMetricsAction(w http.ResponseWriter, req *http
 		}
 	}
 
-	//fmt.Println(min," vs ",max,",",rangeFrom,rangeTo,"range hours:",hours)
+	timeout := h.GetParameterOrDefault(req, "timeout", "60s")
-
+	du, err := time.ParseDuration(timeout)
-	//metrics:=h.GetClusterMetrics(id,bucketSize,min,max)
+	if err != nil {
-	isOverview := h.GetIntOrDefault(req, "overview", 0)
+		log.Error(err)
+		h.WriteError(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), du)
+	defer cancel()
 	var metrics interface{}
-	if isOverview == 1 {
+	if util.StringInArray([]string{IndexThroughputMetricKey, SearchThroughputMetricKey, IndexLatencyMetricKey, SearchLatencyMetricKey}, key) {
-		metrics = h.GetClusterIndexMetrics(id, bucketSize, min, max)
+		metrics = h.GetClusterIndexMetrics(ctx, id, bucketSize, min, max, key)
 	} else {
 		if meta != nil && meta.Config.MonitorConfigs != nil && meta.Config.MonitorConfigs.ClusterStats.Enabled && meta.Config.MonitorConfigs.ClusterStats.Interval != "" {
 			du, _ := time.ParseDuration(meta.Config.MonitorConfigs.ClusterStats.Interval)
@@ -534,7 +540,7 @@ func (h *APIHandler) HandleClusterMetricsAction(w http.ResponseWriter, req *http
 				bucketSize = int(du.Seconds())
 			}
 		}
-		metrics = h.GetClusterMetrics(id, bucketSize, min, max)
+		metrics = h.GetClusterMetrics(ctx, id, bucketSize, min, max, key)
 	}
 
 	resBody["metrics"] = metrics
@@ -546,48 +552,6 @@ func (h *APIHandler) HandleClusterMetricsAction(w http.ResponseWriter, req *http
 
 }
 
-func (h *APIHandler) HandleNodeMetricsAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
-	resBody := map[string]interface{}{}
-	id := ps.ByName("id")
-	bucketSize, min, max, err := h.getMetricRangeAndBucketSize(req, 10, 90)
-	if err != nil {
-		log.Error(err)
-		resBody["error"] = err
-		h.WriteJSON(w, resBody, http.StatusInternalServerError)
-		return
-	}
-	meta := elastic.GetMetadata(id)
-	if meta != nil && meta.Config.MonitorConfigs != nil && meta.Config.MonitorConfigs.NodeStats.Interval != "" {
-		du, _ := time.ParseDuration(meta.Config.MonitorConfigs.NodeStats.Interval)
-		if bucketSize < int(du.Seconds()) {
-			bucketSize = int(du.Seconds())
-		}
-	}
-	nodeName := h.Get(req, "node_name", "")
-	top := h.GetIntOrDefault(req, "top", 5)
-	resBody["metrics"], err = h.getNodeMetrics(id, bucketSize, min, max, nodeName, top)
-	if err != nil {
-		log.Error(err)
-		h.WriteError(w, err.Error(), http.StatusInternalServerError)
-		return
-	}
-	ver := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).GetVersion()
-	if ver.Distribution == "" {
-		cr, err := util.VersionCompare(ver.Number, "6.1")
-		if err != nil {
-			log.Error(err)
-		}
-		if cr < 0 {
-			resBody["tips"] = "The system cluster version is lower than 6.1, the top node may be inaccurate"
-		}
-	}
-
-	err = h.WriteJSON(w, resBody, http.StatusOK)
-	if err != nil {
-		log.Error(err)
-	}
-}
-
 func (h *APIHandler) HandleIndexMetricsAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
 	resBody := map[string]interface{}{}
 	id := ps.ByName("id")
@@ -607,52 +571,73 @@ func (h *APIHandler) HandleIndexMetricsAction(w http.ResponseWriter, req *http.R
 	}
 	indexName := h.Get(req, "index_name", "")
 	top := h.GetIntOrDefault(req, "top", 5)
-	metrics := h.getIndexMetrics(req, id, bucketSize, min, max, indexName, top)
+	key := h.GetParameter(req, "key")
-	if metrics["doc_count"] != nil && metrics["docs_deleted"] != nil && len(metrics["doc_count"].Lines) > 0 && len(metrics["docs_deleted"].Lines) > 0 {
+	timeout := h.GetParameterOrDefault(req, "timeout", "60s")
-		metricA := metrics["doc_count"]
+	du, err := time.ParseDuration(timeout)
-		metricB := metrics["docs_deleted"]
+	if err != nil {
-		if dataA, ok := metricA.Lines[0].Data.([][]interface{}); ok {
+		log.Error(err)
-			if dataB, ok := metricB.Lines[0].Data.([][]interface{}); ok {
+		h.WriteError(w, err.Error(), http.StatusInternalServerError)
-				data := make([]map[string]interface{}, 0, len(dataA)*2)
+		return
-				var (
+	}
-					x1 float64
+	ctx, cancel := context.WithTimeout(context.Background(), du)
-					x2 float64
+	defer cancel()
-				)
+	var metrics map[string]*common.MetricItem
-				for i := 0; i < len(dataA); i++ {
+	if key == DocPercentMetricKey {
-					x1 = dataA[i][1].(float64)
+		metrics = h.getIndexMetrics(ctx, req, id, bucketSize, min, max, indexName, top, DocCountMetricKey)
-					x2 = dataB[i][1].(float64)
+		docsDeletedMetrics := h.getIndexMetrics(ctx, req, id, bucketSize, min, max, indexName, top, DocsDeletedMetricKey)
-					if x1+x2 == 0 {
+		for k, v := range docsDeletedMetrics {
-						continue
+			if v != nil {
-					}
+				metrics[k] = v
-					data = append(data, map[string]interface{}{
-						"x": dataA[i][0],
-						"y": x1 / (x1 + x2) * 100,
-						"g": "Doc Count",
-					})
-					data = append(data, map[string]interface{}{
-						"x": dataA[i][0],
-						"y": x2 / (x1 + x2) * 100,
-						"g": "Doc Deleted",
-					})
-				}
-				metricDocPercent := &common.MetricItem{
-					Axis:  []*common.MetricAxis{},
-					Key:   "doc_percent",
-					Group: metricA.Group,
-					Order: 18,
-					Lines: []*common.MetricLine{
-						{
-							TimeRange: metricA.Lines[0].TimeRange,
-							Data:      data,
-							Type:      common.GraphTypeBar,
-						},
-					},
-				}
-				metrics["doc_percent"] = metricDocPercent
 			}
 		}
-
+		if metrics["doc_count"] != nil && metrics["docs_deleted"] != nil && len(metrics["doc_count"].Lines) > 0 && len(metrics["docs_deleted"].Lines) > 0 {
+			metricA := metrics["doc_count"]
+			metricB := metrics["docs_deleted"]
+			if dataA, ok := metricA.Lines[0].Data.([][]interface{}); ok {
+				if dataB, ok := metricB.Lines[0].Data.([][]interface{}); ok {
+					data := make([]map[string]interface{}, 0, len(dataA)*2)
+					var (
+						x1 float64
+						x2 float64
+					)
+					for i := 0; i < len(dataA); i++ {
+						x1 = dataA[i][1].(float64)
+						x2 = dataB[i][1].(float64)
+						if x1+x2 == 0 {
+							continue
+						}
+						data = append(data, map[string]interface{}{
+							"x": dataA[i][0],
+							"y": x1 / (x1 + x2) * 100,
+							"g": "Doc Count",
+						})
+						data = append(data, map[string]interface{}{
+							"x": dataA[i][0],
+							"y": x2 / (x1 + x2) * 100,
+							"g": "Doc Deleted",
+						})
+					}
+					metricDocPercent := &common.MetricItem{
+						Axis:  []*common.MetricAxis{},
+						Key:   "doc_percent",
+						Group: metricA.Group,
+						Order: 18,
+						Lines: []*common.MetricLine{
+							{
+								TimeRange: metricA.Lines[0].TimeRange,
+								Data:      data,
+								Type:      common.GraphTypeBar,
+							},
+						},
+					}
+					metrics["doc_percent"] = metricDocPercent
+				}
+			}
+		}
+	}else{
+		metrics =  h.getIndexMetrics(ctx, req, id, bucketSize, min, max, indexName, top, key)
 	}
+
 	resBody["metrics"] = metrics
 	ver := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).GetVersion()
 	if ver.Distribution == "" {
@@ -670,42 +655,6 @@ func (h *APIHandler) HandleIndexMetricsAction(w http.ResponseWriter, req *http.R
 		log.Error(err)
 	}
 }
-func (h *APIHandler) HandleQueueMetricsAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
-	resBody := map[string]interface{}{}
-	id := ps.ByName("id")
-	bucketSize, min, max, err := h.getMetricRangeAndBucketSize(req, 10, 90)
-	if err != nil {
-		log.Error(err)
-		resBody["error"] = err
-		h.WriteJSON(w, resBody, http.StatusInternalServerError)
-		return
-	}
-	nodeName := h.Get(req, "node_name", "")
-	top := h.GetIntOrDefault(req, "top", 5)
-	meta := elastic.GetMetadata(id)
-	if meta != nil && meta.Config.MonitorConfigs != nil && meta.Config.MonitorConfigs.NodeStats.Interval != "" {
-		du, _ := time.ParseDuration(meta.Config.MonitorConfigs.NodeStats.Interval)
-		if bucketSize < int(du.Seconds()) {
-			bucketSize = int(du.Seconds())
-		}
-	}
-	resBody["metrics"] = h.getThreadPoolMetrics(id, bucketSize, min, max, nodeName, top)
-	ver := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).GetVersion()
-	if ver.Distribution == "" {
-		cr, err := util.VersionCompare(ver.Number, "6.1")
-		if err != nil {
-			log.Error(err)
-		}
-		if cr < 0 {
-			resBody["tips"] = "The system cluster version is lower than 6.1, the top node may be inaccurate"
-		}
-	}
-
-	err = h.WriteJSON(w, resBody, http.StatusOK)
-	if err != nil {
-		log.Error(err)
-	}
-}
 
 // TODO, use expired hash
 var clusters = map[string]elastic.ElasticsearchConfig{}
@@ -810,56 +759,45 @@ const (
 	CircuitBreakerGroupKey = "circuit_breaker"
 )
 
-func (h *APIHandler) GetClusterMetrics(id string, bucketSize int, min, max int64) map[string]*common.MetricItem {
+const (
+	ClusterStorageMetricKey = "cluster_storage"
+	ClusterDocumentsMetricKey = "cluster_documents"
+	ClusterIndicesMetricKey  = "cluster_indices"
+	ClusterNodeCountMetricKey = "node_count"
+	ClusterHealthMetricKey = "cluster_health"
+	ShardCountMetricKey = "shard_count"
+	CircuitBreakerMetricKey = "circuit_breaker"
+)
+func (h *APIHandler) GetClusterMetrics(ctx context.Context, id string, bucketSize int, min, max int64, metricKey string) map[string]*common.MetricItem {
 
+	var clusterMetricsResult = map[string]*common.MetricItem {}
+	switch metricKey {
+	case ClusterDocumentsMetricKey,
+		ClusterStorageMetricKey,
+		ClusterIndicesMetricKey,
+		ClusterNodeCountMetricKey:
+		clusterMetricsResult = h.getClusterMetricsByKey(ctx, id, bucketSize, min, max, metricKey)
+	case IndexLatencyMetricKey, IndexThroughputMetricKey, SearchThroughputMetricKey, SearchLatencyMetricKey:
+		clusterMetricsResult = h.GetClusterIndexMetrics(ctx, id, bucketSize, min, max, metricKey)
+	case ClusterHealthMetricKey:
+		statusMetric, err := h.getClusterStatusMetric(ctx, id, min, max, bucketSize)
+		if err == nil {
+			clusterMetricsResult[ClusterHealthMetricKey] = statusMetric
+		} else {
+			log.Error("get cluster status metric error: ", err)
+		}
+	case ShardCountMetricKey:
+		clusterMetricsResult = h.getShardsMetric(ctx, id, min, max, bucketSize)
+
+	case CircuitBreakerMetricKey:
+		clusterMetricsResult = h.getCircuitBreakerMetric(ctx, id, min, max, bucketSize)
+	}
+
+	return clusterMetricsResult
+}
+
+func (h *APIHandler) getClusterMetricsByKey(ctx context.Context, id string, bucketSize int, min, max int64, metricKey string) map[string]*common.MetricItem {
 	bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
-
-	clusterMetricItems := []*common.MetricItem{}
-	metricItem := newMetricItem("cluster_storage", 8, StorageGroupKey)
-	metricItem.AddAxi("indices_storage", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
-	metricItem.AddAxi("available_storage", "group2", common.PositionRight, "bytes", "0.[0]", "0.[0]", 5, true)
-
-	metricItem.AddLine("Disk", "Indices Storage", "", "group1", "payload.elasticsearch.cluster_stats.indices.store.size_in_bytes", "max", bucketSizeStr, "", "bytes", "0,0.[00]", "0,0.[00]", false, false)
-	metricItem.AddLine("Disk", "Available Disk", "", "group2", "payload.elasticsearch.cluster_stats.nodes.fs.available_in_bytes", "max", bucketSizeStr, "", "bytes", "0,0.[00]", "0,0.[00]", false, false)
-
-	clusterMetricItems = append(clusterMetricItems, metricItem)
-
-	metricItem = newMetricItem("cluster_documents", 4, StorageGroupKey)
-	metricItem.AddAxi("count", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
-	metricItem.AddAxi("deleted", "group2", common.PositionRight, "num", "0,0", "0,0.[00]", 5, false)
-	metricItem.AddLine("Documents Count", "Documents Count", "", "group1", "payload.elasticsearch.cluster_stats.indices.docs.count", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false)
-	metricItem.AddLine("Documents Deleted", "Documents Deleted", "", "group2", "payload.elasticsearch.cluster_stats.indices.docs.deleted", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false)
-	clusterMetricItems = append(clusterMetricItems, metricItem)
-
-	metricItem = newMetricItem("cluster_indices", 6, StorageGroupKey)
-	metricItem.AddAxi("count", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
-	metricItem.AddLine("Indices Count", "Indices Count", "", "group1", "payload.elasticsearch.cluster_stats.indices.count", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false)
-	clusterMetricItems = append(clusterMetricItems, metricItem)
-
-	metricItem = newMetricItem("node_count", 5, MemoryGroupKey)
-	metricItem.AddAxi("count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
-	meta := elastic.GetMetadata(id)
-	if meta == nil {
-		err := fmt.Errorf("metadata of cluster [%s] is not found", id)
-		panic(err)
-	}
-	majorVersion := meta.GetMajorVersion()
-
-	metricItem.AddLine("Total", "Total Nodes", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.total", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
-
-	//TODO check version difference
-	if majorVersion < 5 {
-		metricItem.AddLine("Master Only", "Master Only", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.master_only", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
-		metricItem.AddLine("Data Node", "Data Only", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.data_only", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
-		metricItem.AddLine("Master Data", "Master Data", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.master_data", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
-	} else {
-		metricItem.AddLine("Master Node", "Master Node", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.master", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
-		metricItem.AddLine("Data Node", "Data Node", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.data", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
-		metricItem.AddLine("Coordinating Node Only", "Coordinating Node Only", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.coordinating_only", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
-		metricItem.AddLine("Ingest Node", "Ingest Node", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.ingest", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
-	}
-
-	clusterMetricItems = append(clusterMetricItems, metricItem)
 	query := map[string]interface{}{}
 	query["query"] = util.MapStr{
 		"bool": util.MapStr{
@@ -898,80 +836,117 @@ func (h *APIHandler) GetClusterMetrics(id string, bucketSize int, min, max int64
 			},
 		},
 	}
-	//todo: since there is four queries, we can change these query to async
+	clusterMetricItems := []*common.MetricItem{}
-	indexMetricsResult := h.GetClusterIndexMetrics(id, bucketSize, min, max)
+	switch metricKey {
-	clusterMetricsResult := h.getSingleMetrics(clusterMetricItems, query, bucketSize)
+	case ClusterStorageMetricKey:
-	for k, v := range clusterMetricsResult {
+		metricItem := newMetricItem(ClusterStorageMetricKey, 8, StorageGroupKey)
-		indexMetricsResult[k] = v
+		metricItem.AddAxi("indices_storage", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
-	}
+		metricItem.AddAxi("available_storage", "group2", common.PositionRight, "bytes", "0.[0]", "0.[0]", 5, true)
-	statusMetric, err := h.getClusterStatusMetric(id, min, max, bucketSize)
-	if err == nil {
-		indexMetricsResult["cluster_health"] = statusMetric
-	} else {
-		log.Error("get cluster status metric error: ", err)
-	}
-	clusterHealthMetricsResult := h.getShardsMetric(id, min, max, bucketSize)
-	for k, v := range clusterHealthMetricsResult {
-		indexMetricsResult[k] = v
-	}
-	// get CircuitBreaker metric
-	circuitBreakerMetricsResult := h.getCircuitBreakerMetric(id, min, max, bucketSize)
-	for k, v := range circuitBreakerMetricsResult {
-		indexMetricsResult[k] = v
-	}
 
-	return indexMetricsResult
+		metricItem.AddLine("Disk", "Indices Storage", "", "group1", "payload.elasticsearch.cluster_stats.indices.store.size_in_bytes", "max", bucketSizeStr, "", "bytes", "0,0.[00]", "0,0.[00]", false, false)
+		metricItem.AddLine("Disk", "Available Disk", "", "group2", "payload.elasticsearch.cluster_stats.nodes.fs.available_in_bytes", "max", bucketSizeStr, "", "bytes", "0,0.[00]", "0,0.[00]", false, false)
+
+		clusterMetricItems = append(clusterMetricItems, metricItem)
+	case ClusterDocumentsMetricKey:
+		metricItem := newMetricItem(ClusterDocumentsMetricKey, 4, StorageGroupKey)
+		metricItem.AddAxi("count", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
+		metricItem.AddAxi("deleted", "group2", common.PositionRight, "num", "0,0", "0,0.[00]", 5, false)
+		metricItem.AddLine("Documents Count", "Documents Count", "", "group1", "payload.elasticsearch.cluster_stats.indices.docs.count", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false)
+		metricItem.AddLine("Documents Deleted", "Documents Deleted", "", "group2", "payload.elasticsearch.cluster_stats.indices.docs.deleted", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false)
+		clusterMetricItems = append(clusterMetricItems, metricItem)
+	case ClusterIndicesMetricKey:
+		metricItem := newMetricItem(ClusterIndicesMetricKey, 6, StorageGroupKey)
+		metricItem.AddAxi("count", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
+		metricItem.AddLine("Indices Count", "Indices Count", "", "group1", "payload.elasticsearch.cluster_stats.indices.count", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false)
+		clusterMetricItems = append(clusterMetricItems, metricItem)
+	case ClusterNodeCountMetricKey:
+		metricItem := newMetricItem("node_count", 5, MemoryGroupKey)
+		metricItem.AddAxi("count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
+		meta := elastic.GetMetadata(id)
+		if meta == nil {
+			err := fmt.Errorf("metadata of cluster [%s] is not found", id)
+			panic(err)
+		}
+		majorVersion := meta.GetMajorVersion()
+
+		metricItem.AddLine("Total", "Total Nodes", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.total", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
+
+		if majorVersion < 5 {
+			metricItem.AddLine("Master Only", "Master Only", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.master_only", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
+			metricItem.AddLine("Data Node", "Data Only", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.data_only", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
+			metricItem.AddLine("Master Data", "Master Data", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.master_data", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
+		} else {
+			metricItem.AddLine("Master Node", "Master Node", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.master", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
+			metricItem.AddLine("Data Node", "Data Node", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.data", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
+			metricItem.AddLine("Coordinating Node Only", "Coordinating Node Only", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.coordinating_only", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
+			metricItem.AddLine("Ingest Node", "Ingest Node", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.ingest", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false)
+		}
+		clusterMetricItems = append(clusterMetricItems, metricItem)
+	}
+	return h.getSingleMetrics(ctx, clusterMetricItems, query, bucketSize)
 }
-func (h *APIHandler) GetClusterIndexMetrics(id string, bucketSize int, min, max int64) map[string]*common.MetricItem {
+const (
+	IndexThroughputMetricKey = "index_throughput"
+	SearchThroughputMetricKey = "search_throughput"
+	IndexLatencyMetricKey    = "index_latency"
+	SearchLatencyMetricKey   = "search_latency"
+)
+func (h *APIHandler) GetClusterIndexMetrics(ctx context.Context, id string, bucketSize int, min, max int64, metricKey string) map[string]*common.MetricItem {
 	bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
 	metricItems := []*common.MetricItem{}
-	metricItem := newMetricItem("index_throughput", 2, OperationGroupKey)
+	switch metricKey {
-	metricItem.AddAxi("indexing", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
+	case IndexThroughputMetricKey:
-	metricItem.AddLine("Indexing Rate", "Total Indexing", "Number of documents being indexed for primary and replica shards.", "group1", "payload.elasticsearch.index_stats.total.indexing.index_total", "max", bucketSizeStr, "doc/s", "num", "0,0.[00]", "0,0.[00]", false, true)
+		metricItem := newMetricItem(IndexThroughputMetricKey, 2, OperationGroupKey)
-	metricItem.AddLine("Indexing Rate", "Primary Indexing", "Number of documents being indexed for primary shards.", "group1", "payload.elasticsearch.index_stats.primaries.indexing.index_total", "max", bucketSizeStr, "doc/s", "num", "0,0.[00]", "0,0.[00]", false, true)
+		metricItem.AddAxi("indexing", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
-	metricItems = append(metricItems, metricItem)
+		metricItem.AddLine("Indexing Rate", "Total Indexing", "Number of documents being indexed for primary and replica shards.", "group1", "payload.elasticsearch.index_stats.total.indexing.index_total", "max", bucketSizeStr, "doc/s", "num", "0,0.[00]", "0,0.[00]", false, true)
+		metricItem.AddLine("Indexing Rate", "Primary Indexing", "Number of documents being indexed for primary shards.", "group1", "payload.elasticsearch.index_stats.primaries.indexing.index_total", "max", bucketSizeStr, "doc/s", "num", "0,0.[00]", "0,0.[00]", false, true)
+		metricItems = append(metricItems, metricItem)
+	case SearchThroughputMetricKey:
+		metricItem := newMetricItem(SearchThroughputMetricKey, 2, OperationGroupKey)
+		metricItem.AddAxi("searching", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
+		metricItem.AddLine("Search Rate", "Total Query",
+			"Number of search requests being executed across primary and replica shards. A single search can run against multiple shards!",
+			"group1", "payload.elasticsearch.index_stats.total.search.query_total", "max", bucketSizeStr, "query/s", "num", "0,0.[00]", "0,0.[00]", false, true)
+		metricItems = append(metricItems, metricItem)
+	case IndexLatencyMetricKey:
+		metricItem := newMetricItem(IndexLatencyMetricKey, 3, LatencyGroupKey)
+		metricItem.AddAxi("indexing", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
 
-	metricItem = newMetricItem("search_throughput", 2, OperationGroupKey)
+		metricItem.AddLine("Indexing", "Indexing Latency", "Average latency for indexing documents.", "group1", "payload.elasticsearch.index_stats.primaries.indexing.index_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
-	metricItem.AddAxi("searching", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
+		metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.index_stats.primaries.indexing.index_total"
-	metricItem.AddLine("Search Rate", "Total Query",
+		metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
-		"Number of search requests being executed across primary and replica shards. A single search can run against multiple shards!",
+			return value / value2
-		"group1", "payload.elasticsearch.index_stats.total.search.query_total", "max", bucketSizeStr, "query/s", "num", "0,0.[00]", "0,0.[00]", false, true)
+		}
-	metricItems = append(metricItems, metricItem)
+		metricItem.AddLine("Indexing", "Delete Latency", "Average latency for delete documents.", "group1", "payload.elasticsearch.index_stats.primaries.indexing.delete_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
+		metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.index_stats.primaries.indexing.delete_total"
+		metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
+			return value / value2
+		}
+		metricItems = append(metricItems, metricItem)
+	case SearchLatencyMetricKey:
+		metricItem := newMetricItem(SearchLatencyMetricKey, 3, LatencyGroupKey)
+		metricItem.AddAxi("searching", "group2", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
 
-	metricItem = newMetricItem("index_latency", 3, LatencyGroupKey)
+		metricItem.AddLine("Searching", "Query Latency", "Average latency for searching query.", "group2", "payload.elasticsearch.index_stats.total.search.query_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
-	metricItem.AddAxi("indexing", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
+		metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.index_stats.total.search.query_total"
+		metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
+			return value / value2
+		}
+		metricItem.AddLine("Searching", "Fetch Latency", "Average latency for searching fetch.", "group2", "payload.elasticsearch.index_stats.total.search.fetch_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
+		metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.index_stats.total.search.fetch_total"
+		metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
+			return value / value2
+		}
+		metricItem.AddLine("Searching", "Scroll Latency", "Average latency for searching fetch.", "group2", "payload.elasticsearch.index_stats.total.search.scroll_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
+		metricItem.Lines[2].Metric.Field2 = "payload.elasticsearch.index_stats.total.search.scroll_total"
+		metricItem.Lines[2].Metric.Calc = func(value, value2 float64) float64 {
+			return value / value2
+		}
+		metricItems = append(metricItems, metricItem)
+	default:
+		panic("not support metric key: " + metricKey)
+	}
 
-	metricItem.AddLine("Indexing", "Indexing Latency", "Average latency for indexing documents.", "group1", "payload.elasticsearch.index_stats.primaries.indexing.index_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
-	metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.index_stats.primaries.indexing.index_total"
-	metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
-		return value / value2
-	}
-	metricItem.AddLine("Indexing", "Delete Latency", "Average latency for delete documents.", "group1", "payload.elasticsearch.index_stats.primaries.indexing.delete_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
-	metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.index_stats.primaries.indexing.delete_total"
-	metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
-		return value / value2
-	}
-	metricItems = append(metricItems, metricItem)
-
-	metricItem = newMetricItem("search_latency", 3, LatencyGroupKey)
-	metricItem.AddAxi("searching", "group2", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
-
-	metricItem.AddLine("Searching", "Query Latency", "Average latency for searching query.", "group2", "payload.elasticsearch.index_stats.total.search.query_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
-	metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.index_stats.total.search.query_total"
-	metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
-		return value / value2
-	}
-	metricItem.AddLine("Searching", "Fetch Latency", "Average latency for searching fetch.", "group2", "payload.elasticsearch.index_stats.total.search.fetch_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
-	metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.index_stats.total.search.fetch_total"
-	metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
-		return value / value2
-	}
-	metricItem.AddLine("Searching", "Scroll Latency", "Average latency for searching fetch.", "group2", "payload.elasticsearch.index_stats.total.search.scroll_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
-	metricItem.Lines[2].Metric.Field2 = "payload.elasticsearch.index_stats.total.search.scroll_total"
-	metricItem.Lines[2].Metric.Calc = func(value, value2 float64) float64 {
-		return value / value2
-	}
-	metricItems = append(metricItems, metricItem)
 	query := map[string]interface{}{}
 	query["query"] = util.MapStr{
 		"bool": util.MapStr{
@@ -1017,10 +992,10 @@ func (h *APIHandler) GetClusterIndexMetrics(id string, bucketSize int, min, max
 			},
 		},
 	}
-	return h.getSingleMetrics(metricItems, query, bucketSize)
+	return h.getSingleMetrics(ctx, metricItems, query, bucketSize)
 }
 
-func (h *APIHandler) getShardsMetric(id string, min, max int64, bucketSize int) map[string]*common.MetricItem {
+func (h *APIHandler) getShardsMetric(ctx context.Context, id string, min, max int64, bucketSize int) map[string]*common.MetricItem {
 	bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
 	query := util.MapStr{
 		"query": util.MapStr{
@@ -1079,10 +1054,10 @@ func (h *APIHandler) getShardsMetric(id string, min, max int64, bucketSize int)
 	metricItem.AddLine("Delayed Unassigned Shards", "Delayed Unassigned Shards", "", "group1", "payload.elasticsearch.cluster_health.delayed_unassigned_shards", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false)
 	var clusterHealthMetrics []*common.MetricItem
 	clusterHealthMetrics = append(clusterHealthMetrics, metricItem)
-	return h.getSingleMetrics(clusterHealthMetrics, query, bucketSize)
+	return h.getSingleMetrics(ctx, clusterHealthMetrics, query, bucketSize)
 }
 
-func (h *APIHandler) getCircuitBreakerMetric(id string, min, max int64, bucketSize int) map[string]*common.MetricItem {
+func (h *APIHandler) getCircuitBreakerMetric(ctx context.Context, id string, min, max int64, bucketSize int) map[string]*common.MetricItem {
 	bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
 	query := util.MapStr{
 		"query": util.MapStr{
@@ -1140,10 +1115,10 @@ func (h *APIHandler) getCircuitBreakerMetric(id string, min, max int64, bucketSi
 	metricItem.AddLine("In Flight Requests Breaker Tripped", "In Flight Requests Tripped", "", "group1", "payload.elasticsearch.node_stats.breakers.in_flight_requests.tripped", "sum", bucketSizeStr, "times/s", "num", "0,0.[00]", "0,0.[00]", false, true)
 	var circuitBreakerMetrics []*common.MetricItem
 	circuitBreakerMetrics = append(circuitBreakerMetrics, metricItem)
-	return h.getSingleMetrics(circuitBreakerMetrics, query, bucketSize)
+	return h.getSingleMetrics(ctx, circuitBreakerMetrics, query, bucketSize)
 }
 
-func (h *APIHandler) getClusterStatusMetric(id string, min, max int64, bucketSize int) (*common.MetricItem, error) {
+func (h *APIHandler) getClusterStatusMetric(ctx context.Context, id string, min, max int64, bucketSize int) (*common.MetricItem, error) {
 	bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
 	intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr)
 	if err != nil {
@@ -1204,7 +1179,8 @@ func (h *APIHandler) getClusterStatusMetric(id string, min, max int64, bucketSiz
 			},
 		},
 	}
-	response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query))
+	queryDSL := util.MustToJSONBytes(query)
+	response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).QueryDSL(ctx, getAllMetricsIndex(), nil, util.MustToJSONBytes(query))
 	if err != nil {
 		log.Error(err)
 		return nil, err
@@ -1221,6 +1197,7 @@ func (h *APIHandler) getClusterStatusMetric(id string, min, max int64, bucketSiz
 	}
 	metricItem.Lines[0].Data = metricData
 	metricItem.Lines[0].Type = common.GraphTypeBar
+	metricItem.Request = string(queryDSL)
 	return metricItem, nil
 }
 

modules/elastic/api/v1/metrics_util.go

@@ -24,6 +24,7 @@
 package v1
 
 import (
+	"context"
 	"fmt"
 	"infini.sh/framework/core/env"
 	"net/http"
@@ -109,9 +110,10 @@ func generateGroupAggs(nodeMetricItems []GroupMetricItem) map[string]interface{}
 	return aggs
 }
 
-func (h *APIHandler) getMetrics(query map[string]interface{}, grpMetricItems []GroupMetricItem, bucketSize int) map[string]*common.MetricItem {
+func (h *APIHandler) getMetrics(ctx context.Context, query map[string]interface{}, grpMetricItems []GroupMetricItem, bucketSize int) map[string]*common.MetricItem {
 	bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
-	response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query))
+	queryDSL := util.MustToJSONBytes(query)
+	response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).QueryDSL(ctx, getAllMetricsIndex(), nil,  queryDSL)
 	if err != nil {
 		log.Error(err)
 		panic(err)
@@ -205,6 +207,7 @@ func (h *APIHandler) getMetrics(query map[string]interface{}, grpMetricItems []G
 			}
 			line.Data = grpMetricData[dataKey][line.Metric.Label]
 		}
+		metricItem.MetricItem.Request = string(queryDSL)
 		result[metricItem.Key] = metricItem.MetricItem
 	}
 	return result
@@ -328,7 +331,7 @@ func GetMetricRangeAndBucketSize(minStr string, maxStr string, bucketSize int, m
 }
 
 // 获取单个指标，可以包含多条曲线
-func (h *APIHandler) getSingleMetrics(metricItems []*common.MetricItem, query map[string]interface{}, bucketSize int) map[string]*common.MetricItem {
+func (h *APIHandler) getSingleMetrics(ctx context.Context, metricItems []*common.MetricItem, query map[string]interface{}, bucketSize int) map[string]*common.MetricItem {
 	metricData := map[string][][]interface{}{}
 
 	aggs := map[string]interface{}{}
@@ -387,7 +390,8 @@ func (h *APIHandler) getSingleMetrics(metricItems []*common.MetricItem, query ma
 			"aggs": aggs,
 		},
 	}
-	response, err := elastic.GetClient(clusterID).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query))
+	queryDSL := util.MustToJSONBytes(query)
+	response, err := elastic.GetClient(clusterID).QueryDSL(ctx, getAllMetricsIndex(), nil, queryDSL)
 	if err != nil {
 		log.Error(err)
 		panic(err)
@@ -449,6 +453,7 @@ func (h *APIHandler) getSingleMetrics(metricItems []*common.MetricItem, query ma
 			line.TimeRange = common.TimeRange{Min: minDate, Max: maxDate}
 			line.Data = metricData[line.Metric.GetDataKey()]
 		}
+		metricItem.Request = string(queryDSL)
 		result[metricItem.Key] = metricItem
 	}
 

modules/elastic/api/v1/node_overview.go

@@ -28,6 +28,7 @@
 package v1
 
 import (
+	"context"
 	"fmt"
 	log "github.com/cihub/seelog"
 	httprouter "infini.sh/framework/core/api/router"
@@ -410,7 +411,7 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps
 			},
 		},
 	}
-	metrics := h.getMetrics(query, nodeMetricItems, bucketSize)
+	metrics := h.getMetrics(context.Background(), query, nodeMetricItems, bucketSize)
 	indexMetrics := map[string]util.MapStr{}
 	for key, item := range metrics {
 		for _, line := range item.Lines {
@@ -692,7 +693,7 @@ func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Reque
 	metricItem =newMetricItem("parent_breaker", 8, SystemGroupKey)
 	metricItem.AddLine("Parent Breaker Tripped","Parent Breaker Tripped","Rate of the circuit breaker has been triggered and prevented an out of memory error.","group1","payload.elasticsearch.node_stats.breakers.parent.tripped","max",bucketSizeStr,"times/s","num","0,0.[00]","0,0.[00]",false,true)
 	metricItems=append(metricItems,metricItem)
-	metrics := h.getSingleMetrics(metricItems,query, bucketSize)
+	metrics := h.getSingleMetrics(context.Background(), metricItems,query, bucketSize)
 	healthMetric, err := getNodeHealthMetric(query, bucketSize)
 	if err != nil {
 		log.Error(err)

modules/elastic/api/v1/threadpool_metrics.go

@@ -1,562 +0,0 @@
-// Copyright (C) INFINI Labs & INFINI LIMITED.
-//
-// The INFINI Console is offered under the GNU Affero General Public License v3.0
-// and as commercial software.
-//
-// For commercial licensing, contact us at:
-//   - Website: infinilabs.com
-//   - Email: hello@infini.ltd
-//
-// Open Source licensed under AGPL V3:
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-package v1
-
-import (
-	"fmt"
-	log "github.com/cihub/seelog"
-	"infini.sh/framework/core/elastic"
-	"infini.sh/framework/core/global"
-	"infini.sh/framework/core/util"
-	"infini.sh/framework/modules/elastic/common"
-	"strings"
-)
-
-const (
-	ThreadPoolGetGroupKey    = "thread_pool_get"
-	ThreadPoolSearchGroupKey   = "thread_pool_search"
-	ThreadPoolFlushGroupKey   = "thread_pool_flush"
-	ThreadPoolRefreshGroupKey     = "thread_pool_refresh"
-	ThreadPoolWriteGroupKey      = "thread_pool_write"
-	ThreadPoolForceMergeGroupKey = "thread_pool_force_merge"
-	ThreadPoolIndexGroupKey = "thread_pool_index"
-	ThreadPoolBulkGroupKey  = "thread_pool_bulk"
-)
-
-func (h *APIHandler) getThreadPoolMetrics(clusterID string, bucketSize int, min, max int64, nodeName string, top int) map[string]*common.MetricItem{
-	bucketSizeStr:=fmt.Sprintf("%vs",bucketSize)
-	var must = []util.MapStr{
-		{
-			"term":util.MapStr{
-				"metadata.labels.cluster_id":util.MapStr{
-					"value": clusterID,
-				},
-			},
-		},
-		{
-			"term": util.MapStr{
-				"metadata.category": util.MapStr{
-					"value": "elasticsearch",
-				},
-			},
-		},
-		{
-			"term": util.MapStr{
-				"metadata.name": util.MapStr{
-					"value": "node_stats",
-				},
-			},
-		},
-	}
-	var (
-		nodeNames []string
-		err error
-	)
-	if nodeName != "" {
-		nodeNames = strings.Split(nodeName, ",")
-		top = len(nodeNames)
-	}else{
-		nodeNames, err = h.getTopNodeName(clusterID, top, 15)
-		if err != nil {
-			log.Error(err)
-		}
-	}
-	if len(nodeNames) > 0 {
-		must = append(must, util.MapStr{
-			"terms": util.MapStr{
-				"metadata.labels.transport_address": nodeNames,
-			},
-		})
-	}
-
-	query:=map[string]interface{}{}
-	query["query"]=util.MapStr{
-		"bool": util.MapStr{
-			"must": must,
-			"filter": []util.MapStr{
-				{
-					"range": util.MapStr{
-						"timestamp": util.MapStr{
-							"gte": min,
-							"lte": max,
-						},
-					},
-				},
-			},
-		},
-	}
-	searchThreadsMetric := newMetricItem("search_threads", 1, ThreadPoolSearchGroupKey)
-	searchThreadsMetric.AddAxi("Search Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems := []GroupMetricItem{
-		{
-			Key: "search_threads",
-			Field: "payload.elasticsearch.node_stats.thread_pool.search.threads",
-			ID: util.GetUUID(),
-			IsDerivative: false,
-			MetricItem: searchThreadsMetric,
-			FormatType: "num",
-			Units: "",
-		},
-	}
-	searchQueueMetric := newMetricItem("search_queue", 1, ThreadPoolSearchGroupKey)
-	searchQueueMetric.AddAxi("Search Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "search_queue",
-		Field: "payload.elasticsearch.node_stats.thread_pool.search.queue",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: searchQueueMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	searchActiveMetric := newMetricItem("search_active", 1, ThreadPoolSearchGroupKey)
-	searchActiveMetric.AddAxi("Search Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "search_active",
-		Field: "payload.elasticsearch.node_stats.thread_pool.search.active",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: searchActiveMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	searchRejectedMetric := newMetricItem("search_rejected", 1, ThreadPoolSearchGroupKey)
-	searchRejectedMetric.AddAxi("Search Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "search_rejected",
-		Field: "payload.elasticsearch.node_stats.thread_pool.search.rejected",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: searchRejectedMetric,
-		FormatType: "num",
-		Units: "rejected/s",
-	})
-
-	getThreadsMetric := newMetricItem("get_threads", 1, ThreadPoolGetGroupKey)
-	getThreadsMetric.AddAxi("Get Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "get_threads",
-		Field: "payload.elasticsearch.node_stats.thread_pool.get.threads",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: getThreadsMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	getQueueMetric := newMetricItem("get_queue", 1, ThreadPoolGetGroupKey)
-	getQueueMetric.AddAxi("Get Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "get_queue",
-		Field: "payload.elasticsearch.node_stats.thread_pool.get.queue",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: getQueueMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	getActiveMetric := newMetricItem("get_active", 1, ThreadPoolGetGroupKey)
-	getActiveMetric.AddAxi("Get Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "get_active",
-		Field: "payload.elasticsearch.node_stats.thread_pool.get.active",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: getActiveMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	getRejectedMetric := newMetricItem("get_rejected", 1, ThreadPoolGetGroupKey)
-	getRejectedMetric.AddAxi("Get Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "get_rejected",
-		Field: "payload.elasticsearch.node_stats.thread_pool.get.rejected",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: getRejectedMetric,
-		FormatType: "num",
-		Units: "rejected/s",
-	})
-
-	flushThreadsMetric := newMetricItem("flush_threads", 1, ThreadPoolFlushGroupKey)
-	flushThreadsMetric.AddAxi("Flush Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "flush_threads",
-		Field: "payload.elasticsearch.node_stats.thread_pool.flush.threads",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: flushThreadsMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	flushQueueMetric := newMetricItem("flush_queue", 1, ThreadPoolFlushGroupKey)
-	flushQueueMetric.AddAxi("Get Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "flush_queue",
-		Field: "payload.elasticsearch.node_stats.thread_pool.flush.queue",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: flushQueueMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	flushActiveMetric := newMetricItem("flush_active", 1, ThreadPoolFlushGroupKey)
-	flushActiveMetric.AddAxi("Flush Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "flush_active",
-		Field: "payload.elasticsearch.node_stats.thread_pool.flush.active",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: flushActiveMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	flushRejectedMetric := newMetricItem("flush_rejected", 1, ThreadPoolFlushGroupKey)
-	flushRejectedMetric.AddAxi("Flush Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "flush_rejected",
-		Field: "payload.elasticsearch.node_stats.thread_pool.flush.rejected",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: flushRejectedMetric,
-		FormatType: "num",
-		Units: "rejected/s",
-	})
-
-	majorVersion := elastic.GetMetadata(clusterID).GetMajorVersion()
-	ver := elastic.GetClient(clusterID).GetVersion()
-
-	if (ver.Distribution == "" || ver.Distribution == elastic.Elasticsearch) && majorVersion < 6{
-		indexThreadsMetric := newMetricItem("index_threads", 1, ThreadPoolIndexGroupKey)
-		indexThreadsMetric.AddAxi("Index Threads Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
-
-		queueMetricItems = append(queueMetricItems, GroupMetricItem{
-			Key:          "index_threads",
-			Field:        "payload.elasticsearch.node_stats.thread_pool.index.threads",
-			ID:           util.GetUUID(),
-			IsDerivative: false,
-			MetricItem:   indexThreadsMetric,
-			FormatType:   "num",
-			Units:        "",
-		})
-		indexQueueMetric := newMetricItem("index_queue", 1, ThreadPoolIndexGroupKey)
-		indexQueueMetric.AddAxi("Index Queue Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
-
-		queueMetricItems = append(queueMetricItems, GroupMetricItem{
-			Key:          "index_queue",
-			Field:        "payload.elasticsearch.node_stats.thread_pool.index.queue",
-			ID:           util.GetUUID(),
-			IsDerivative: false,
-			MetricItem:   indexQueueMetric,
-			FormatType:   "num",
-			Units:        "",
-		})
-		indexActiveMetric := newMetricItem("index_active", 1, ThreadPoolIndexGroupKey)
-		indexActiveMetric.AddAxi("Index Active Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
-
-		queueMetricItems = append(queueMetricItems, GroupMetricItem{
-			Key:          "index_active",
-			Field:        "payload.elasticsearch.node_stats.thread_pool.index.active",
-			ID:           util.GetUUID(),
-			IsDerivative: false,
-			MetricItem:   indexActiveMetric,
-			FormatType:   "num",
-			Units:        "",
-		})
-		indexRejectedMetric := newMetricItem("index_rejected", 1, ThreadPoolIndexGroupKey)
-		indexRejectedMetric.AddAxi("Index Rejected Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
-
-		queueMetricItems = append(queueMetricItems, GroupMetricItem{
-			Key:          "index_rejected",
-			Field:        "payload.elasticsearch.node_stats.thread_pool.index.rejected",
-			ID:           util.GetUUID(),
-			IsDerivative: true,
-			MetricItem:   indexRejectedMetric,
-			FormatType:   "num",
-			Units:        "rejected/s",
-		})
-
-		bulkThreadsMetric := newMetricItem("bulk_threads", 1, ThreadPoolBulkGroupKey)
-		bulkThreadsMetric.AddAxi("Bulk Threads Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
-
-		queueMetricItems = append(queueMetricItems, GroupMetricItem{
-			Key:          "bulk_threads",
-			Field:        "payload.elasticsearch.node_stats.thread_pool.bulk.threads",
-			ID:           util.GetUUID(),
-			IsDerivative: false,
-			MetricItem:   bulkThreadsMetric,
-			FormatType:   "num",
-			Units:        "",
-		})
-		bulkQueueMetric := newMetricItem("bulk_queue", 1, ThreadPoolBulkGroupKey)
-		bulkQueueMetric.AddAxi("Bulk Queue Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
-
-		queueMetricItems = append(queueMetricItems, GroupMetricItem{
-			Key:          "bulk_queue",
-			Field:        "payload.elasticsearch.node_stats.thread_pool.bulk.queue",
-			ID:           util.GetUUID(),
-			IsDerivative: false,
-			MetricItem:   bulkQueueMetric,
-			FormatType:   "num",
-			Units:        "",
-		})
-		bulkActiveMetric := newMetricItem("bulk_active", 1, ThreadPoolBulkGroupKey)
-		bulkActiveMetric.AddAxi("Bulk Active Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
-
-		queueMetricItems = append(queueMetricItems, GroupMetricItem{
-			Key:          "bulk_active",
-			Field:        "payload.elasticsearch.node_stats.thread_pool.bulk.active",
-			ID:           util.GetUUID(),
-			IsDerivative: false,
-			MetricItem:   bulkActiveMetric,
-			FormatType:   "num",
-			Units:        "",
-		})
-		bulkRejectedMetric := newMetricItem("bulk_rejected", 1, ThreadPoolBulkGroupKey)
-		bulkRejectedMetric.AddAxi("Bulk Rejected Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
-
-		queueMetricItems = append(queueMetricItems, GroupMetricItem{
-			Key:          "bulk_rejected",
-			Field:        "payload.elasticsearch.node_stats.thread_pool.bulk.rejected",
-			ID:           util.GetUUID(),
-			IsDerivative: true,
-			MetricItem:   bulkRejectedMetric,
-			FormatType:   "num",
-			Units:        "rejected/s",
-		})
-	}else {
-		writeThreadsMetric := newMetricItem("write_threads", 1, ThreadPoolWriteGroupKey)
-		writeThreadsMetric.AddAxi("Write Threads Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
-
-		queueMetricItems = append(queueMetricItems, GroupMetricItem{
-			Key:          "write_threads",
-			Field:        "payload.elasticsearch.node_stats.thread_pool.write.threads",
-			ID:           util.GetUUID(),
-			IsDerivative: false,
-			MetricItem:   writeThreadsMetric,
-			FormatType:   "num",
-			Units:        "",
-		})
-		writeQueueMetric := newMetricItem("write_queue", 1, ThreadPoolWriteGroupKey)
-		writeQueueMetric.AddAxi("Write Queue Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
-
-		queueMetricItems = append(queueMetricItems, GroupMetricItem{
-			Key:          "write_queue",
-			Field:        "payload.elasticsearch.node_stats.thread_pool.write.queue",
-			ID:           util.GetUUID(),
-			IsDerivative: false,
-			MetricItem:   writeQueueMetric,
-			FormatType:   "num",
-			Units:        "",
-		})
-		writeActiveMetric := newMetricItem("write_active", 1, ThreadPoolWriteGroupKey)
-		writeActiveMetric.AddAxi("Write Active Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
-
-		queueMetricItems = append(queueMetricItems, GroupMetricItem{
-			Key:          "write_active",
-			Field:        "payload.elasticsearch.node_stats.thread_pool.write.active",
-			ID:           util.GetUUID(),
-			IsDerivative: false,
-			MetricItem:   writeActiveMetric,
-			FormatType:   "num",
-			Units:        "",
-		})
-		writeRejectedMetric := newMetricItem("write_rejected", 1, ThreadPoolWriteGroupKey)
-		writeRejectedMetric.AddAxi("Write Rejected Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
-
-		queueMetricItems = append(queueMetricItems, GroupMetricItem{
-			Key:          "write_rejected",
-			Field:        "payload.elasticsearch.node_stats.thread_pool.write.rejected",
-			ID:           util.GetUUID(),
-			IsDerivative: true,
-			MetricItem:   writeRejectedMetric,
-			FormatType:   "num",
-			Units:        "rejected/s",
-		})
-	}
-	refreshThreadsMetric := newMetricItem("refresh_threads", 1, ThreadPoolRefreshGroupKey)
-	refreshThreadsMetric.AddAxi("Refresh Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "refresh_threads",
-		Field: "payload.elasticsearch.node_stats.thread_pool.refresh.threads",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: refreshThreadsMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	refreshQueueMetric := newMetricItem("refresh_queue", 1, ThreadPoolRefreshGroupKey)
-	refreshQueueMetric.AddAxi("Refresh Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "refresh_queue",
-		Field: "payload.elasticsearch.node_stats.thread_pool.refresh.queue",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: refreshQueueMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	refreshActiveMetric := newMetricItem("refresh_active", 1, ThreadPoolRefreshGroupKey)
-	refreshActiveMetric.AddAxi("Refresh Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "refresh_active",
-		Field: "payload.elasticsearch.node_stats.thread_pool.refresh.active",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: refreshActiveMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	refreshRejectedMetric := newMetricItem("refresh_rejected", 1, ThreadPoolRefreshGroupKey)
-	refreshRejectedMetric.AddAxi("Refresh Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "refresh_rejected",
-		Field: "payload.elasticsearch.node_stats.thread_pool.refresh.rejected",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: refreshRejectedMetric,
-		FormatType: "num",
-		Units: "rejected/s",
-	})
-	forceMergeThreadsMetric := newMetricItem("force_merge_threads", 1, ThreadPoolForceMergeGroupKey)
-	forceMergeThreadsMetric.AddAxi("Force Merge Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "force_merge_threads",
-		Field: "payload.elasticsearch.node_stats.thread_pool.force_merge.threads",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: forceMergeThreadsMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	forceMergeQueueMetric := newMetricItem("force_merge_queue", 1, ThreadPoolForceMergeGroupKey)
-	forceMergeQueueMetric.AddAxi("Force Merge Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "force_merge_queue",
-		Field: "payload.elasticsearch.node_stats.thread_pool.force_merge.queue",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: forceMergeQueueMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	forceMergeActiveMetric := newMetricItem("force_merge_active", 1, ThreadPoolForceMergeGroupKey)
-	forceMergeActiveMetric.AddAxi("Force Merge Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "force_merge_active",
-		Field: "payload.elasticsearch.node_stats.thread_pool.force_merge.active",
-		ID: util.GetUUID(),
-		IsDerivative: false,
-		MetricItem: forceMergeActiveMetric,
-		FormatType: "num",
-		Units: "",
-	})
-	forceMergeRejectedMetric := newMetricItem("force_merge_rejected", 1, ThreadPoolForceMergeGroupKey)
-	forceMergeRejectedMetric.AddAxi("Force Merge Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true)
-
-	queueMetricItems = append(queueMetricItems, GroupMetricItem{
-		Key: "force_merge_rejected",
-		Field: "payload.elasticsearch.node_stats.thread_pool.force_merge.rejected",
-		ID: util.GetUUID(),
-		IsDerivative: true,
-		MetricItem: forceMergeRejectedMetric,
-		FormatType: "num",
-		Units: "rejected/s",
-	})
-	//Get Thread Pool queue
-	aggs:=map[string]interface{}{}
-
-	for _,metricItem:=range queueMetricItems{
-		aggs[metricItem.ID]=util.MapStr{
-			"max":util.MapStr{
-				"field": metricItem.Field,
-			},
-		}
-		if metricItem.Field2 != "" {
-			aggs[metricItem.ID + "_field2"]=util.MapStr{
-				"max":util.MapStr{
-					"field": metricItem.Field2,
-				},
-			}
-		}
-
-		if metricItem.IsDerivative{
-			aggs[metricItem.ID+"_deriv"]=util.MapStr{
-				"derivative":util.MapStr{
-					"buckets_path": metricItem.ID,
-				},
-			}
-			if metricItem.Field2 != "" {
-				aggs[metricItem.ID + "_field2_deriv"]=util.MapStr{
-					"derivative":util.MapStr{
-						"buckets_path": metricItem.ID + "_field2",
-					},
-				}
-			}
-		}
-	}
-	intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr)
-	if err != nil {
-		log.Error(err)
-		panic(err)
-	}
-
-	query["size"]=0
-	query["aggs"]= util.MapStr{
-		"group_by_level": util.MapStr{
-			"terms": util.MapStr{
-				"field": "metadata.labels.transport_address",
-				"size":  top,
-			},
-			"aggs": util.MapStr{
-				"dates": util.MapStr{
-					"date_histogram":util.MapStr{
-						"field": "timestamp",
-						intervalField: bucketSizeStr,
-					},
-					"aggs":aggs,
-				},
-			},
-		},
-	}
-	return h.getMetrics(query, queueMetricItems, bucketSize)
-}