feat: support querying top N metrics in the Insight Data Query API (#74)

* feat: refactor the Insight data API to support metric aggregation for multiple data inputs

* fix:  bucket sorting issue when a sub-aggregation contains a `date_histogram`

* feat: add insight metric CRUD api

* fix: empty value of metric

* fix: incorrect grouping results

* feat: add builtin metric template

* fix: bucket sort when aggregation with P99

* chore: update group size

* chore: update release notes

* chore: update release notes

* chore: update release notes

* chore: update group size

config/setup/common/insight.tpl

@@ -14,7 +14,7 @@ POST $[[SETUP_INDEX_PREFIX]]widget/$[[SETUP_DOC_TYPE]]/cji1sc28go5i051pl1i0
           "formula": "a",
           "items": [
             {
-              "field": "*",
+              "field": "id",
               "name": "a",
               "statistic": "count"
             }
@@ -51,7 +51,7 @@ POST $[[SETUP_INDEX_PREFIX]]widget/$[[SETUP_DOC_TYPE]]/cji1ttq8go5i051pl1t2
           "formula": "a",
           "items": [
             {
-              "field": "*",
+              "field": "id",
               "name": "a",
               "statistic": "count"
             }
@@ -94,7 +94,7 @@ POST $[[SETUP_INDEX_PREFIX]]widget/$[[SETUP_DOC_TYPE]]/cji1ttq8go5i051pl1t1
           ],
           "items": [
             {
-              "field": "*",
+              "field": "id",
               "name": "a",
               "statistic": "count"
             }
@@ -137,7 +137,7 @@ POST $[[SETUP_INDEX_PREFIX]]widget/$[[SETUP_DOC_TYPE]]/cji1ttq8go5i051pl1t0
           ],
           "items": [
             {
-              "field": "*",
+              "field": "id",
               "name": "a",
               "statistic": "count"
             }
@@ -3566,4 +3566,579 @@ POST $[[SETUP_INDEX_PREFIX]]layout/$[[SETUP_DOC_TYPE]]/cicmhbt3q95ich72lrvg
     },
     "type": "workspace",
     "is_fixed": true
+}
+
+#shard level
+PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH9
+{
+  "id": "bD2jH5QB7KvGccywNCH9",
+  "name": "Indexing Rate",
+  "key": "indexing_rate",
+  "level": "shard",
+  "formula": "bD2jH5QB7KvGccywNCH9/{{.bucket_size_in_second}}",
+  "items": [
+    {
+      "name": "bD2jH5QB7KvGccywNCH9",
+      "field": "payload.elasticsearch.shard_stats.indexing.index_total",
+      "statistic": "rate"
+    }
+  ],
+  "statistics": ["rate"],
+  "format": "number",
+  "unit": "doc/s",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+
+PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH1
+{
+  "id": "bD2jH5QB7KvGccywNCH1",
+  "name": "Shard Storage",
+  "key": "shard_storage",
+  "level": "shard",
+  "formula": "bD2jH5QB7KvGccywNCH1",
+  "items": [
+    {
+      "name": "bD2jH5QB7KvGccywNCH1",
+      "field": "payload.elasticsearch.shard_stats.store.size_in_bytes",
+      "statistic": "max"
+    }
+  ],
+  "statistics": ["max", "min", "sum", "avg", "p99", "medium"],
+  "format": "bytes",
+  "unit": "",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH5
+{
+  "id": "bD2jH5QB7KvGccywNCH5",
+  "name": "Document Count",
+  "key": "doc_count",
+  "level": "shard",
+  "formula": "bD2jH5QB7KvGccywNCH5",
+  "items": [
+    {
+      "name": "bD2jH5QB7KvGccywNCH5",
+      "field": "payload.elasticsearch.shard_stats.docs.count",
+      "statistic": "max"
+    }
+  ],
+  "statistics": ["max", "min", "sum", "avg", "p99", "medium"],
+  "format": "number",
+  "unit": "",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+
+PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH2
+{
+  "id": "bD2jH5QB7KvGccywNCH2",
+  "name": "Search Rate",
+  "key": "search_rate",
+  "level": "shard",
+  "formula": "bD2jH5QB7KvGccywNCH2/{{.bucket_size_in_second}}",
+  "items": [
+    {
+      "name": "bD2jH5QB7KvGccywNCH2",
+      "field": "payload.elasticsearch.shard_stats.search.query_total",
+      "statistic": "rate"
+    }
+  ],
+  "statistics": ["rate"],
+  "format": "number",
+  "unit": "doc/s",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH3
+{
+  "id": "bD2jH5QB7KvGccywNCH3",
+  "name": "Indexing Latency",
+  "key": "indexing_latency",
+  "level": "shard",
+  "formula": "bD2jH5QB7KvGccywNCx3/bD2jH5QB7KvGccywNCH3",
+  "items": [
+    {
+      "name": "bD2jH5QB7KvGccywNCx3",
+      "field": "payload.elasticsearch.shard_stats.indexing.index_total",
+      "statistic": "rate"
+    },
+     {
+      "name": "bD2jH5QB7KvGccywNCH3",
+      "field": "payload.elasticsearch.shard_stats.indexing.index_time_in_millis",
+      "statistic": "rate"
+    }
+  ],
+  "statistics": ["rate"],
+  "format": "number",
+  "unit": "ms",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+
+PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH4
+{
+  "id": "bD2jH5QB7KvGccywNCH4",
+  "name": "Search Latency",
+  "key": "search_latency",
+  "level": "shard",
+  "formula": "bD2jH5QB7KvGccywNCx4/bD2jH5QB7KvGccywNCH4",
+  "items": [
+    {
+      "name": "bD2jH5QB7KvGccywNCH4",
+      "field": "payload.elasticsearch.shard_stats.search.query_total",
+      "statistic": "rate"
+    },
+     {
+      "name": "bD2jH5QB7KvGccywNCx4",
+      "field": "payload.elasticsearch.shard_stats.search.query_time_in_millis",
+      "statistic": "rate"
+    }
+  ],
+  "statistics": ["rate"],
+  "format": "number",
+  "unit": "ms",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH6
+{
+  "id": "bD2jH5QB7KvGccywNCH6",
+  "name": "Segment Count",
+  "key": "segment_count",
+  "level": "shard",
+  "formula": "bD2jH5QB7KvGccywNCH6",
+  "items": [
+    {
+      "name": "bD2jH5QB7KvGccywNCH6",
+      "field": "payload.elasticsearch.shard_stats.segments.count",
+      "statistic": "max"
+    }
+  ],
+  "statistics": ["max", "min", "sum", "avg", "p99", "medium"],
+  "format": "number",
+  "unit": "",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+
+PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH7
+{
+  "id": "bD2jH5QB7KvGccywNCH7",
+  "name": "Segment memory",
+  "key": "segment_memory",
+  "level": "shard",
+  "formula": "bD2jH5QB7KvGccywNCH7",
+  "items": [
+    {
+      "name": "bD2jH5QB7KvGccywNCH7",
+      "field": "payload.elasticsearch.shard_stats.segments.memory_in_bytes",
+      "statistic": "max"
+    }
+  ],
+  "statistics": ["max", "min", "sum", "avg", "p99", "medium"],
+  "format": "number",
+  "unit": "",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+
+#indices level
+PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH9
+{
+  "id": "aD2jH5QB7KvGccywNCH9",
+  "name": "Indexing Rate",
+  "key": "indexing_rate",
+  "level": "indices",
+  "formula": "aD2jH5QB7KvGccywNCH9/{{.bucket_size_in_second}}",
+  "items": [
+    {
+      "name": "aD2jH5QB7KvGccywNCH9",
+      "field": "payload.elasticsearch.index_stats.primaries.indexing.index_total",
+      "statistic": "rate"
+    }
+  ],
+  "statistics": ["rate"],
+  "format": "number",
+  "unit": "doc/s",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+
+PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH1
+{
+  "id": "aD2jH5QB7KvGccywNCH1",
+  "name": "Index Storage",
+  "key": "index_storage",
+  "level": "indices",
+  "formula": "aD2jH5QB7KvGccywNCH1",
+  "items": [
+    {
+      "name": "aD2jH5QB7KvGccywNCH1",
+      "field": "payload.elasticsearch.index_stats.total.store.size_in_bytes",
+      "statistic": "max"
+    }
+  ],
+  "statistics": ["max", "min", "sum", "avg", "p99", "medium"],
+  "format": "bytes",
+  "unit": "",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH5
+{
+  "id": "aD2jH5QB7KvGccywNCH5",
+  "name": "Document Count",
+  "key": "doc_count",
+  "level": "indices",
+  "formula": "aD2jH5QB7KvGccywNCH5",
+  "items": [
+    {
+      "name": "aD2jH5QB7KvGccywNCH5",
+      "field": "payload.elasticsearch.index_stats.total.docs.count",
+      "statistic": "max"
+    }
+  ],
+  "statistics": ["max", "min", "sum", "avg", "p99", "medium"],
+  "format": "number",
+  "unit": "",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+
+PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH2
+{
+  "id": "aD2jH5QB7KvGccywNCH2",
+  "name": "Search Rate",
+  "key": "search_rate",
+  "level": "indices",
+  "formula": "aD2jH5QB7KvGccywNCH2/{{.bucket_size_in_second}}",
+  "items": [
+    {
+      "name": "aD2jH5QB7KvGccywNCH2",
+      "field": "payload.elasticsearch.index_stats.total.search.query_total",
+      "statistic": "rate"
+    }
+  ],
+  "statistics": ["rate"],
+  "format": "number",
+  "unit": "doc/s",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH3
+{
+  "id": "aD2jH5QB7KvGccywNCH3",
+  "name": "Indexing Latency",
+  "key": "indexing_latency",
+  "level": "indices",
+  "formula": "aD2jH5QB7KvGccywNCx3/aD2jH5QB7KvGccywNCH3",
+  "items": [
+    {
+      "name": "aD2jH5QB7KvGccywNCH3",
+      "field": "payload.elasticsearch.index_stats.primaries.indexing.index_total",
+      "statistic": "rate"
+    },
+     {
+      "name": "aD2jH5QB7KvGccywNCx3",
+      "field": "payload.elasticsearch.index_stats.primaries.indexing.index_time_in_millis",
+      "statistic": "rate"
+    }
+  ],
+  "statistics": ["rate"],
+  "format": "number",
+  "unit": "ms",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+
+PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH4
+{
+  "id": "aD2jH5QB7KvGccywNCH4",
+  "name": "Search Latency",
+  "key": "search_latency",
+  "level": "indices",
+  "formula": "aD2jH5QB7KvGccywNCx4/aD2jH5QB7KvGccywNCH4",
+  "items": [
+    {
+      "name": "aD2jH5QB7KvGccywNCH4",
+      "field": "payload.elasticsearch.index_stats.total.search.query_total",
+      "statistic": "rate"
+    },
+     {
+      "name": "aD2jH5QB7KvGccywNCx4",
+      "field": "payload.elasticsearch.index_stats.total.search.query_time_in_millis",
+      "statistic": "rate"
+    }
+  ],
+  "statistics": ["rate"],
+  "format": "number",
+  "unit": "ms",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH6
+{
+  "id": "aD2jH5QB7KvGccywNCH6",
+  "name": "Segment Count",
+  "key": "segment_count",
+  "level": "indices",
+  "formula": "aD2jH5QB7KvGccywNCH6",
+  "items": [
+    {
+      "name": "aD2jH5QB7KvGccywNCH6",
+      "field": "payload.elasticsearch.index_stats.total.segments.count",
+      "statistic": "max"
+    }
+  ],
+  "statistics": ["max", "min", "sum", "avg", "p99", "medium"],
+  "format": "number",
+  "unit": "",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+
+PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH7
+{
+  "id": "aD2jH5QB7KvGccywNCH7",
+  "name": "Segment memory",
+  "key": "segment_memory",
+  "level": "indices",
+  "formula": "aD2jH5QB7KvGccywNCH7",
+  "items": [
+    {
+      "name": "aD2jH5QB7KvGccywNCH7",
+      "field": "payload.elasticsearch.index_stats.total.segments.memory_in_bytes",
+      "statistic": "max"
+    }
+  ],
+  "statistics": ["max", "min", "sum", "avg", "p99", "medium"],
+  "format": "bytes",
+  "unit": "",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+
+#node level
+PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH9
+{
+  "id": "jD2jH5QB7KvGccywNCH9",
+  "name": "Indexing Rate",
+  "key": "indexing_rate",
+  "level": "node",
+  "formula": "jD2jH5QB7KvGccywH9/{{.bucket_size_in_second}}",
+  "items": [
+    {
+      "name": "jD2jH5QB7KvGccywH9",
+      "field": "payload.elasticsearch.node_stats.indices.indexing.index_total",
+      "statistic": "derivative"
+    }
+  ],
+  "statistics": ["rate"],
+  "format": "number",
+  "unit": "doc/s",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+
+
+PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH4
+{
+  "id": "jD2jH5QB7KvGccywNCH4",
+  "name": "Process CPU Usage",
+  "key": "process_cpu_used",
+  "level": "node",
+  "formula": "jD2jH5QB7KvGccywNCH4",
+  "items": [
+    {
+      "name": "jD2jH5QB7KvGccywNCH4",
+      "field": "payload.elasticsearch.node_stats.process.cpu.percent",
+      "statistic": "avg"
+    }
+  ],
+  "statistics": ["max", "min", "sum", "avg", "p99", "medium"],
+  "format": "",
+  "unit": "%",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+
+PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH3
+{
+  "id": "jD2jH5QB7KvGccywNCH3",
+  "name": "JVM Heap Usage",
+  "key": "jvm_heap_used",
+  "level": "node",
+  "formula": "jD2jH5QB7KvGccywNCH3",
+  "items": [
+    {
+      "name": "jD2jH5QB7KvGccywNCH3",
+      "field": "payload.elasticsearch.node_stats.jvm.mem.heap_used_in_bytes",
+      "statistic": "max"
+    }
+  ],
+  "statistics": ["max", "min", "sum", "avg", "p99", "medium"],
+  "format": "bytes",
+  "unit": "",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH1
+{
+  "id": "jD2jH5QB7KvGccywNCH1",
+  "name": "Indexing Latency",
+  "key": "indexing_latency",
+  "level": "node",
+  "formula": "jD2jH5QB7KvGccywNCx1/jD2jH5QB7KvGccywNCH1",
+  "items": [
+    {
+      "name": "jD2jH5QB7KvGccywNCH1",
+      "field": "payload.elasticsearch.node_stats.indices.indexing.index_total",
+      "statistic": "rate"
+    },
+     {
+      "name": "jD2jH5QB7KvGccywNCx1",
+      "field": "payload.elasticsearch.node_stats.indices.indexing.index_time_in_millis",
+      "statistic": "rate"
+    }
+  ],
+  "statistics": ["rate"],
+  "format": "number",
+  "unit": "ms",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+
+PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH0
+{
+  "id": "jD2jH5QB7KvGccywNCH0",
+  "name": "Search Rate",
+  "key": "search_rate",
+  "level": "node",
+  "formula": "jD2jH5QB7KvGccywH0/{{.bucket_size_in_second}}",
+  "items": [
+    {
+      "name": "jD2jH5QB7KvGccywH0",
+      "field": "payload.elasticsearch.node_stats.indices.search.query_total",
+      "statistic": "derivative"
+    }
+  ],
+  "statistics": ["rate"],
+  "format": "number",
+  "unit": "query/s",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+
+PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH9
+{
+  "id": "jD2jH5QB7KvGccywNCH9",
+  "name": "Indexing Rate",
+  "key": "indexing_rate",
+  "level": "node",
+  "formula": "jD2jH5QB7KvGccywH9/{{.bucket_size_in_second}}",
+  "items": [
+    {
+      "name": "jD2jH5QB7KvGccywH9",
+      "field": "payload.elasticsearch.node_stats.indices.indexing.index_total",
+      "statistic": "rate"
+    }
+  ],
+  "statistics": ["rate"],
+  "format": "number",
+  "unit": "doc/s",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH5
+{
+  "id": "jD2jH5QB7KvGccywNCH5",
+  "name": "Search Latency",
+  "key": "search_latency",
+  "level": "node",
+  "formula": "jD2jH5QB7KvGccywNCx5/jD2jH5QB7KvGccywNCH5",
+  "items": [
+    {
+      "name": "jD2jH5QB7KvGccywNCH5",
+      "field": "payload.elasticsearch.node_stats.indices.search.query_total",
+      "statistic": "rate"
+    },
+     {
+      "name": "jD2jH5QB7KvGccywNCx5",
+      "field": "payload.elasticsearch.node_stats.indices.search.query_time_in_millis",
+      "statistic": "rate"
+    }
+  ],
+  "statistics": ["rate"],
+  "format": "number",
+  "unit": "ms",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+
+PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH6
+{
+  "id": "jD2jH5QB7KvGccywNCH6",
+  "name": "Indices Storage",
+  "key": "indices_storage",
+  "level": "node",
+  "formula": "jD2jH5QB7KvGccywNCH6",
+  "items": [
+    {
+      "name": "jD2jH5QB7KvGccywNCH6",
+      "field": "payload.elasticsearch.node_stats.indices.store.size_in_bytes",
+      "statistic": "max"
+    }
+  ],
+  "statistics": ["max", "min", "sum", "avg", "p99", "medium"],
+  "format": "bytes",
+  "unit": "",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
+}
+PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH7
+{
+  "id": "jD2jH5QB7KvGccywNCH7",
+  "name": "Document Count",
+  "key": "doc_count",
+  "level": "node",
+  "formula": "jD2jH5QB7KvGccywNCH7",
+  "items": [
+    {
+      "name": "jD2jH5QB7KvGccywNCH7",
+      "field": "payload.elasticsearch.node_stats.indices.docs.count",
+      "statistic": "max"
+    }
+  ],
+  "statistics": ["max", "min", "sum", "avg", "p99", "medium"],
+  "format": "number",
+  "unit": "",
+  "builtin": true,
+  "created": "2025-01-09T14:30:56.63155+08:00",
+  "updated": "2025-01-09T14:30:56.63155+08:00"
 }

docs/content.en/docs/release-notes/_index.md

@@ -14,7 +14,9 @@ Information about release notes of INFINI Console is provided here.
 ### Features
 - Add allocation to activities if is cluster health change and changed to red.
 - Add index metrics for segment memory (norms, points, version map, fixed bit set).
-
+- Support querying top N metrics in the Insight Data Query API
+- Add insight metric CURD API for managing custom metrics
+- Add built-in metrics templates for common use cases
 ### Bug fix
 - Fixed query thread pool metrics when cluster uuid is empty
 - Fixed unit tests

main.go

@@ -158,6 +158,7 @@ func main() {
 		orm.RegisterSchemaWithIndexName(api3.RemoteConfig{}, "configs")
 		orm.RegisterSchemaWithIndexName(model.AuditLog{}, "audit-logs")
 		orm.RegisterSchemaWithIndexName(host.HostInfo{}, "host")
+		orm.RegisterSchemaWithIndexName(insight.MetricBase{}, "metric")
 
 		module.Start()
 

model/insight/metric_data.go

@@ -29,6 +29,7 @@ package insight
 
 import (
 	"fmt"
+	"infini.sh/framework/core/orm"
 	"infini.sh/framework/core/util"
 	"regexp"
 )
@@ -43,11 +44,35 @@ type Metric struct {
 	Sort []GroupSort `json:"sort,omitempty"`
 	ClusterId string   `json:"cluster_id,omitempty"`
 	Formula string `json:"formula,omitempty"`
+	//array of formula for new version
+	Formulas []string `json:"formulas,omitempty"`
 	Items []MetricItem `json:"items"`
-	FormatType string `json:"format_type,omitempty"`
+	FormatType string `json:"format,omitempty"`
 	TimeFilter interface{} `json:"time_filter,omitempty"`
 	TimeBeforeGroup bool `json:"time_before_group,omitempty"`
 	BucketLabel *BucketLabel `json:"bucket_label,omitempty"`
+	// number of buckets to return, used for aggregation auto_date_histogram when bucket size equals 'auto'
+	Buckets uint `json:"buckets,omitempty"`
+	Unit string `json:"unit,omitempty"`
+}
+
+type MetricBase struct {
+	orm.ORMObjectBase
+	//display name of the metric
+	Name  string `json:"name"`
+	//metric identifier
+	Key   string `json:"key"`
+	//optional values : "node", "indices", "shard"
+	Level string `json:"level"`
+	//metric calculation formula
+	Formula    string       `json:"formula,omitempty"`
+	Items      []MetricItem `json:"items"`
+	FormatType string       `json:"format,omitempty"`
+	Unit       string       `json:"unit,omitempty"`
+	//determine if this metric is built-in
+	Builtin bool `json:"builtin"`
+	//array of supported calculation statistic, eg: "avg", "sum", "min", "max"
+	Statistics []string `json:"statistics,omitempty"`
 }
 
 type GroupSort struct {
@@ -105,12 +130,8 @@ func (m *Metric) ValidateSortKey() error {
 		if !util.StringInArray([]string{"desc", "asc"}, sortItem.Direction){
 			return fmt.Errorf("unknown sort direction [%s]", sortItem.Direction)
 		}
-		if v, ok := mm[sortItem.Key]; !ok && !util.StringInArray([]string{"_key", "_count"}, sortItem.Key){
+		if _, ok := mm[sortItem.Key]; !ok && !util.StringInArray([]string{"_key", "_count"}, sortItem.Key){
 			return fmt.Errorf("unknown sort key [%s]", sortItem.Key)
-		}else{
-			if v != nil && v.Statistic == "derivative" {
-				return fmt.Errorf("can not sort by pipeline agg [%s]", v.Statistic)
-			}
 		}
 	}
 	return nil

plugin/api/insight/api.go

@@ -56,4 +56,7 @@ func InitAPI() {
 	api.HandleAPIMethod(api.POST, "/elasticsearch/:id/map_label/_render", insight.renderMapLabelTemplate)
 	api.HandleAPIMethod(api.GET, "/insight/widget/:widget_id", insight.getWidget)
 	api.HandleAPIMethod(api.POST, "/insight/widget", insight.RequireLogin(insight.createWidget))
+	api.HandleAPIMethod(api.POST, "/insight/metric", insight.createMetric)
+	api.HandleAPIMethod(api.PUT, "/insight/metric/:metric_id", insight.updateMetric)
+	api.HandleAPIMethod(api.DELETE, "/insight/metric/:metric_id", insight.deleteMetric)
 }

plugin/api/insight/metadata.go

@@ -248,8 +248,8 @@ func getMetricData(metric *insight.Metric) (interface{}, error) {
 		return nil, err
 	}
 	esClient := elastic.GetClient(metric.ClusterId)
-	//log.Error(string(util.MustToJSONBytes(query)))
+	queryDSL := util.MustToJSONBytes(query)
-	searchRes, err := esClient.SearchWithRawQueryDSL(metric.IndexPattern, util.MustToJSONBytes(query))
+	searchRes, err := esClient.SearchWithRawQueryDSL(metric.IndexPattern, queryDSL)
 	if err != nil {
 		return nil, err
 	}
@@ -266,83 +266,101 @@ func getMetricData(metric *insight.Metric) (interface{}, error) {
 		}
 	}
 	timeBeforeGroup := metric.AutoTimeBeforeGroup()
-	metricData := CollectMetricData(agg, timeBeforeGroup)
+	metricData, interval := CollectMetricData(agg, timeBeforeGroup)
+	formula := strings.TrimSpace(metric.Formula)
+	//support older versions for a single formula.
+	if metric.Formula != "" && len(metric.Formulas) == 0 {
+		metric.Formulas = []string{metric.Formula}
+	}
 
 	var targetMetricData []insight.MetricData
-	formula := strings.TrimSpace(metric.Formula)
+	if len(metric.Items) == 1 && len(metric.Formulas) == 0 {
-	if len(metric.Items) == 1 && formula == "" {
 		targetMetricData = metricData
 	} else {
-		tpl, err := template.New("insight_formula").Parse(formula)
-		if err != nil {
-			return nil, err
-		}
-		msgBuffer := &bytes.Buffer{}
 		params := map[string]interface{}{}
 		if metric.BucketSize != "" {
-			du, err := util.ParseDuration(metric.BucketSize)
+			bucketSize := metric.BucketSize
-			if err != nil {
+			if  metric.BucketSize == "auto" && interval != "" {
-				return nil, err
+				bucketSize = interval
+			}
+			if interval != "" || bucketSize != "auto" {
+				du, err := util.ParseDuration(bucketSize)
+				if err != nil {
+					return nil, err
+				}
+				params["bucket_size_in_second"] = du.Seconds()
 			}
-			params["bucket_size_in_second"] = du.Seconds()
 		}
-		err = tpl.Execute(msgBuffer, params)
-		if err != nil {
-			return nil, err
-		}
-		formula = msgBuffer.String()
 		for _, md := range metricData {
 			targetData := insight.MetricData{
 				Groups: md.Groups,
 				Data:   map[string][]insight.MetricDataItem{},
 			}
-			expression, err := govaluate.NewEvaluableExpression(formula)
+			retMetricDataItem := insight.MetricDataItem{}
-			if err != nil {
+			for _, formula = range metric.Formulas {
-				return nil, err
+				tpl, err := template.New("insight_formula").Parse(formula)
-			}
-			dataLength := 0
-			for _, v := range md.Data {
-				dataLength = len(v)
-				break
-			}
-		DataLoop:
-			for i := 0; i < dataLength; i++ {
-				parameters := map[string]interface{}{}
-				var timestamp interface{}
-				hasValidData := false
-				for k, v := range md.Data {
-					if len(k) == 20 {
-						continue
-					}
-					if len(v) < dataLength {
-						continue
-					}
-					if _, ok := v[i].Value.(float64); !ok {
-						continue DataLoop
-					}
-					hasValidData = true
-					parameters[k] = v[i].Value
-					timestamp = v[i].Timestamp
-				}
-				//todo return error?
-				if !hasValidData {
-					continue
-				}
-				result, err := expression.Evaluate(parameters)
 				if err != nil {
 					return nil, err
 				}
-				if r, ok := result.(float64); ok {
+				msgBuffer := &bytes.Buffer{}
-					if math.IsNaN(r) || math.IsInf(r, 0) {
+				err = tpl.Execute(msgBuffer, params)
-						//if !isFilterNaN {
+				if err != nil {
-						//	targetData.Data["result"] = append(targetData.Data["result"], []interface{}{timestamp, math.NaN()})
+					return nil, err
-						//}
+				}
+				resolvedFormula := msgBuffer.String()
+				expression, err := govaluate.NewEvaluableExpression(resolvedFormula)
+				if err != nil {
+					return nil, err
+				}
+				dataLength := 0
+				for _, v := range md.Data {
+					dataLength = len(v)
+					break
+				}
+			DataLoop:
+				for i := 0; i < dataLength; i++ {
+					parameters := map[string]interface{}{}
+					var timestamp interface{}
+					hasValidData := false
+					for k, v := range md.Data {
+						if _, ok := v[i].Value.(float64); !ok {
+							continue DataLoop
+						}
+						hasValidData = true
+						parameters[k] = v[i].Value
+						timestamp = v[i].Timestamp
+					}
+					//todo return error?
+					if !hasValidData {
 						continue
 					}
+					result, err := expression.Evaluate(parameters)
+					if err != nil {
+						log.Debugf("evaluate formula error: %v", err)
+						continue
+					}
+					if r, ok := result.(float64); ok {
+						if math.IsNaN(r) || math.IsInf(r, 0) {
+							//if !isFilterNaN {
+							//	targetData.Data["result"] = append(targetData.Data["result"], []interface{}{timestamp, math.NaN()})
+							//}
+							continue
+						}
+					}
+					retMetricDataItem.Timestamp = timestamp
+					if len(metric.Formulas) <= 1 && metric.Formula != ""{
+						//support older versions by returning the result for a single formula.
+						retMetricDataItem.Value = result
+					} else {
+						if v, ok := retMetricDataItem.Value.(map[string]interface{}); ok {
+							v[formula] = result
+						}else{
+							retMetricDataItem.Value = map[string]interface{}{formula: result}
+						}
+					}
 				}
-
-				targetData.Data["result"] = append(targetData.Data["result"], insight.MetricDataItem{Timestamp: timestamp, Value: result})
 			}
+			targetData.Data["result"] = append(targetData.Data["result"], retMetricDataItem)
 			targetMetricData = append(targetMetricData, targetData)
 		}
 	}
@@ -356,7 +374,10 @@ func getMetricData(metric *insight.Metric) (interface{}, error) {
 			}
 		}
 	}
-	return result, nil
+	return util.MapStr{
+		"data": result,
+		"request": string(queryDSL),
+	}, nil
 }
 
 func getMetadataByIndexPattern(clusterID, indexPattern, timeField string, filter interface{}, fieldsFormat map[string]string) (interface{}, error) {

plugin/api/insight/metric.go

@@ -0,0 +1,166 @@
+// Copyright (C) INFINI Labs & INFINI LIMITED.
+//
+// The INFINI Console is offered under the GNU Affero General Public License v3.0
+// and as commercial software.
+//
+// For commercial licensing, contact us at:
+//   - Website: infinilabs.com
+//   - Email: hello@infini.ltd
+//
+// Open Source licensed under AGPL V3:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+/* Copyright © INFINI Ltd. All rights reserved.
+ * web: https://infinilabs.com
+ * mail: hello#infini.ltd */
+
+package insight
+
+import (
+	"errors"
+	log "github.com/cihub/seelog"
+	"infini.sh/console/model/insight"
+	httprouter "infini.sh/framework/core/api/router"
+	"infini.sh/framework/core/orm"
+	"infini.sh/framework/core/util"
+	"infini.sh/framework/modules/elastic"
+	"net/http"
+)
+
+func (h *InsightAPI) createMetric(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+	var obj = &insight.MetricBase{}
+	err := h.DecodeJSON(req, obj)
+	if err != nil {
+		h.WriteError(w, err.Error(), http.StatusInternalServerError)
+		log.Error(err)
+		return
+	}
+	err = orm.Create(nil, obj)
+	if err != nil {
+		h.WriteError(w, err.Error(), http.StatusInternalServerError)
+		log.Error(err)
+		return
+	}
+
+	h.WriteJSON(w, util.MapStr{
+		"_id":    obj.ID,
+		"result": "created",
+	}, 200)
+
+}
+
+func (h *InsightAPI) getMetric(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+	id := ps.MustGetParameter("metric_id")
+
+	obj := insight.MetricBase{}
+	obj.ID = id
+
+	_, err := orm.Get(&obj)
+	if err != nil {
+		if errors.Is(err, elastic.ErrNotFound) {
+			h.WriteJSON(w, util.MapStr{
+				"_id":   id,
+				"found": false,
+			}, http.StatusNotFound)
+			return
+		}
+		h.WriteError(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	h.WriteJSON(w, util.MapStr{
+		"found":   true,
+		"_id":     id,
+		"_source": obj,
+	}, 200)
+}
+
+func (h *InsightAPI) updateMetric(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+	id := ps.MustGetParameter("metric_id")
+	obj := insight.MetricBase{}
+
+	obj.ID = id
+	_, err := orm.Get(&obj)
+	if err != nil {
+		if errors.Is(err, elastic.ErrNotFound) {
+			h.WriteJSON(w, util.MapStr{
+				"_id":   id,
+				"found": false,
+			}, http.StatusNotFound)
+			return
+		}
+		h.WriteError(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	id = obj.ID
+	create := obj.Created
+	obj = insight.MetricBase{}
+	err = h.DecodeJSON(req, &obj)
+	if err != nil {
+		h.WriteError(w, err.Error(), http.StatusInternalServerError)
+		log.Error(err)
+		return
+	}
+	//protect
+	obj.ID = id
+	obj.Created = create
+	err = orm.Update(nil, &obj)
+	if err != nil {
+		h.WriteError(w, err.Error(), http.StatusInternalServerError)
+		log.Error(err)
+		return
+	}
+
+	h.WriteJSON(w, util.MapStr{
+		"_id":    obj.ID,
+		"result": "updated",
+	}, 200)
+}
+
+func (h *InsightAPI) deleteMetric(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+	id := ps.MustGetParameter("metric_id")
+
+	obj := insight.MetricBase{}
+	obj.ID = id
+
+	_, err := orm.Get(&obj)
+	if err != nil {
+		if errors.Is(err, elastic.ErrNotFound) {
+			h.WriteJSON(w, util.MapStr{
+				"_id":   id,
+				"found": false,
+			}, http.StatusNotFound)
+			return
+		}
+		h.WriteError(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	if obj.Builtin {
+		h.WriteError(w, "cannot delete builtin metrics", http.StatusBadRequest)
+		return
+	}
+
+	err = orm.Delete(nil, &obj)
+	if err != nil {
+		h.WriteError(w, err.Error(), http.StatusInternalServerError)
+		log.Error(err)
+		return
+	}
+
+	h.WriteJSON(w, util.MapStr{
+		"_id":    obj.ID,
+		"result": "deleted",
+	}, 200)
+}

plugin/api/insight/metric_util.go

@@ -72,7 +72,7 @@ func generateAgg(metricItem *insight.MetricItem, timeField string) map[string]in
 			"includes": []string{field},
 		}
 		aggValue["sort"] = []util.MapStr{
-			util.MapStr{
+			{
 				timeField: util.MapStr{
 					"order": "desc",
 				},
@@ -114,23 +114,46 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) {
 			return nil, err
 		}
 	}
-	verInfo := elastic.GetClient(metric.ClusterId).GetVersion()
+	var (
+		useDateHistogram = false
+		dateHistogramAgg util.MapStr
+		dateHistogramAggName string
+	)
+	if metric.BucketSize != "" && metric.TimeField != "" {
+		useDateHistogram = true
+		if metric.BucketSize == "auto" {
+			dateHistogramAggName = "auto_date_histogram"
+			buckets := metric.Buckets
+			if buckets == 0 {
+				buckets = 2
+			}
+			dateHistogramAgg = util.MapStr{
+				"field": metric.TimeField,
+				"buckets": buckets,
+			}
+		}else{
+			dateHistogramAggName = "date_histogram"
+			verInfo := elastic.GetClient(metric.ClusterId).GetVersion()
 
-	if verInfo.Number == "" {
+			if verInfo.Number == "" {
-		panic("invalid version")
+				panic("invalid version")
+			}
+
+			intervalField, err := elastic.GetDateHistogramIntervalField(verInfo.Distribution, verInfo.Number, metric.BucketSize)
+			if err != nil {
+				return nil, fmt.Errorf("get interval field error: %w", err)
+			}
+			dateHistogramAgg = util.MapStr{
+				"field": metric.TimeField,
+				intervalField: metric.BucketSize,
+			}
+		}
 	}
 
-	intervalField, err := elastic.GetDateHistogramIntervalField(verInfo.Distribution, verInfo.Number, metric.BucketSize)
+	if useDateHistogram && !timeBeforeGroup {
-	if err != nil {
-		return nil, fmt.Errorf("get interval field error: %w", err)
-	}
-	if metric.BucketSize != "" && !timeBeforeGroup {
 		basicAggs = util.MapStr{
 			"time_buckets": util.MapStr{
-				"date_histogram": util.MapStr{
+				dateHistogramAggName: dateHistogramAgg,
-					"field":       metric.TimeField,
-					intervalField: metric.BucketSize,
-				},
 				"aggs": basicAggs,
 			},
 		}
@@ -138,7 +161,7 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) {
 
 	var rootAggs util.MapStr
 	groups := metric.Groups
-	err = metric.ValidateSortKey()
+	err := metric.ValidateSortKey()
 	if err != nil {
 		return nil, err
 	}
@@ -156,12 +179,43 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) {
 				"field": groups[i].Field,
 				"size":  limit,
 			}
-			if i == grpLength-1 && len(metric.Sort) > 0 {
+			if i == grpLength - 1 && len(metric.Sort) > 0 {
-				var termsOrder []interface{}
+				//use bucket sort instead of terms order when time after group
-				for _, sortItem := range metric.Sort {
+				if !timeBeforeGroup && len(metric.Sort) > 0 {
-					termsOrder = append(termsOrder, util.MapStr{sortItem.Key: sortItem.Direction})
+					basicAggs["sort_field"] = util.MapStr{
+						"max_bucket": util.MapStr{
+							"buckets_path": fmt.Sprintf("time_buckets>%s", metric.Sort[0].Key),
+						},
+					}
+					//using 65536 as a workaround for the terms aggregation limit; the actual limit is enforced in the bucket sort step
+					termsCfg["size"] = 65536
+					basicAggs["bucket_sorter"] = util.MapStr{
+						"bucket_sort": util.MapStr{
+							"size": limit,
+							"sort": []util.MapStr{
+								{"sort_field": util.MapStr{"order": metric.Sort[0].Direction}},
+							},
+						},
+					}
+				}else{
+					var termsOrder []interface{}
+					percentAggs := []string{"p99", "p95", "p90", "p80", "p50"}
+					for _, sortItem := range metric.Sort {
+						var percent string
+						for _, item := range metric.Items {
+							lowerCaseStatistic := strings.ToLower(item.Statistic)
+							if item.Name == sortItem.Key && util.StringInArray(percentAggs, lowerCaseStatistic) {
+								percent = lowerCaseStatistic[1:]
+							}
+						}
+						sortKey := sortItem.Key
+						if percent != "" {
+							sortKey = fmt.Sprintf("%s[%s]", sortItem.Key, percent)
+						}
+						termsOrder = append(termsOrder, util.MapStr{sortKey: sortItem.Direction})
+					}
+					termsCfg["order"] = termsOrder
 				}
-				termsCfg["order"] = termsOrder
 			}
 			groupAgg := util.MapStr{
 				"terms": termsCfg,
@@ -176,32 +230,26 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) {
 			}
 			lastGroupAgg = groupAgg
 		}
-		if metric.BucketSize == "" || (metric.BucketSize != "" && !timeBeforeGroup) {
+		if useDateHistogram && timeBeforeGroup {
-			rootAggs = util.MapStr{
-				util.GetUUID(): lastGroupAgg,
-			}
-		} else {
 			rootAggs = util.MapStr{
 				"time_buckets": util.MapStr{
-					"date_histogram": util.MapStr{
+					dateHistogramAggName: dateHistogramAgg,
-						"field":       metric.TimeField,
-						intervalField: metric.BucketSize,
-					},
 					"aggs": util.MapStr{
 						util.GetUUID(): lastGroupAgg,
 					},
 				},
 			}
+		} else {
+			rootAggs = util.MapStr{
+				util.GetUUID(): lastGroupAgg,
+			}
 		}
 
 	} else {
 		if metric.BucketSize != "" && timeBeforeGroup {
 			basicAggs = util.MapStr{
 				"time_buckets": util.MapStr{
-					"date_histogram": util.MapStr{
+					dateHistogramAggName: dateHistogramAgg,
-						"field":       metric.TimeField,
-						intervalField: metric.BucketSize,
-					},
 					"aggs": basicAggs,
 				},
 			}
@@ -228,20 +276,22 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) {
 	return queryDsl, nil
 }
 
-func CollectMetricData(agg interface{}, timeBeforeGroup bool) []insight.MetricData {
+func CollectMetricData(agg interface{}, timeBeforeGroup bool) ([]insight.MetricData, string) {
 	metricData := []insight.MetricData{}
+	var interval string
 	if timeBeforeGroup {
-		collectMetricDataOther(agg, nil, &metricData, nil)
+		interval = collectMetricDataOther(agg, nil, &metricData, nil)
 	} else {
-		collectMetricData(agg, nil, &metricData)
+		interval = collectMetricData(agg, nil, &metricData)
 	}
-	return metricData
+	return metricData, interval
 }
 
 // timeBeforeGroup => false
-func collectMetricData(agg interface{}, groupValues []string, metricData *[]insight.MetricData) {
+func collectMetricData(agg interface{}, groupValues []string, metricData *[]insight.MetricData) (interval string){
 	if aggM, ok := agg.(map[string]interface{}); ok {
 		if timeBks, ok := aggM["time_buckets"].(map[string]interface{}); ok {
+			interval, _ = timeBks["interval"].(string)
 			if bks, ok := timeBks["buckets"].([]interface{}); ok {
 				md := insight.MetricData{
 					Data:   map[string][]insight.MetricDataItem{},
@@ -255,7 +305,7 @@ func collectMetricData(agg interface{}, groupValues []string, metricData *[]insi
 								continue
 							}
 
-							if vm, ok := v.(map[string]interface{}); ok && len(k) < 5 {
+							if vm, ok := v.(map[string]interface{}); ok {
 								collectMetricDataItem(k, vm, &md, bkM["key"])
 							}
 
@@ -283,14 +333,12 @@ func collectMetricData(agg interface{}, groupValues []string, metricData *[]insi
 								newGroupValues := make([]string, 0, len(groupValues)+1)
 								newGroupValues = append(newGroupValues, groupValues...)
 								newGroupValues = append(newGroupValues, currentGroup)
-								collectMetricData(bk, newGroupValues, metricData)
+								interval = collectMetricData(bk, newGroupValues, metricData)
 							}
 						}
 					} else {
 						//non time series metric data
-						if len(k) < 5 {
+						collectMetricDataItem(k, vm, &md, nil)
-							collectMetricDataItem(k, vm, &md, nil)
-						}
 					}
 				}
 			}
@@ -299,12 +347,14 @@ func collectMetricData(agg interface{}, groupValues []string, metricData *[]insi
 			}
 		}
 	}
+	return
 }
 
 // timeBeforeGroup => true
-func collectMetricDataOther(agg interface{}, groupValues []string, metricData *[]insight.MetricData, timeKey interface{}) {
+func collectMetricDataOther(agg interface{}, groupValues []string, metricData *[]insight.MetricData, timeKey interface{}) (interval string){
 	if aggM, ok := agg.(map[string]interface{}); ok {
 		if timeBks, ok := aggM["time_buckets"].(map[string]interface{}); ok {
+			interval, _ = timeBks["interval"].(string)
 			if bks, ok := timeBks["buckets"].([]interface{}); ok {
 				md := insight.MetricData{
 					Data:   map[string][]insight.MetricDataItem{},
@@ -318,7 +368,7 @@ func collectMetricDataOther(agg interface{}, groupValues []string, metricData *[
 							}
 							if vm, ok := v.(map[string]interface{}); ok {
 								if vm["buckets"] != nil {
-									collectMetricDataOther(vm, groupValues, metricData, bkM["key"])
+									interval = collectMetricDataOther(vm, groupValues, metricData, bkM["key"])
 								} else {
 									collectMetricDataItem(k, vm, &md, bkM["key"])
 								}
@@ -346,7 +396,7 @@ func collectMetricDataOther(agg interface{}, groupValues []string, metricData *[
 						newGroupValues := make([]string, 0, len(groupValues)+1)
 						newGroupValues = append(newGroupValues, groupValues...)
 						newGroupValues = append(newGroupValues, currentGroup)
-						collectMetricDataOther(bk, newGroupValues, metricData, timeKey)
+						interval = collectMetricDataOther(bk, newGroupValues, metricData, timeKey)
 					}
 				}
 			} else {
@@ -354,7 +404,7 @@ func collectMetricDataOther(agg interface{}, groupValues []string, metricData *[
 				for k, v := range aggM {
 					if vm, ok := v.(map[string]interface{}); ok {
 						if vm["buckets"] != nil {
-							collectMetricDataOther(vm, groupValues, metricData, timeKey)
+							interval = collectMetricDataOther(vm, groupValues, metricData, timeKey)
 						} else {
 							collectMetricDataItem(k, vm, &md, timeKey)
 						}
@@ -367,6 +417,7 @@ func collectMetricDataOther(agg interface{}, groupValues []string, metricData *[
 			}
 		}
 	}
+	return
 }
 
 func collectMetricDataItem(key string, vm map[string]interface{}, metricData *insight.MetricData, timeKey interface{}) {

plugin/api/platform/domain.go

@@ -31,6 +31,7 @@ import (
 	"infini.sh/console/core/security/enum"
 	consoleModel "infini.sh/console/model"
 	"infini.sh/console/model/alerting"
+	"infini.sh/console/model/insight"
 	"infini.sh/framework/core/elastic"
 	"infini.sh/framework/core/event"
 	"infini.sh/framework/core/model"
@@ -211,6 +212,10 @@ func GetCollectionMetas() map[string]CollectionMeta {
 				},
 				MatchObject: &alerting.Rule{},
 			},
+			"metric": {
+				Name: "metric",
+				MatchObject: &insight.MetricBase{},
+			},
 		}
 	})
 	return collectionMetas