feat: support querying top N metrics in the Insight Data Query API (#74)
* feat: refactor the Insight data API to support metric aggregation for multiple data inputs * fix: bucket sorting issue when a sub-aggregation contains a `date_histogram` * feat: add insight metric CRUD api * fix: empty value of metric * fix: incorrect grouping results * feat: add builtin metric template * fix: bucket sort when aggregation with P99 * chore: update group size * chore: update release notes * chore: update release notes * chore: update release notes * chore: update group size
This commit is contained in:
silenceqi
GitHub
parent
f8eb0c2fc7
commit
692b87eb7a
|
|
@ -14,7 +14,7 @@ POST $[[SETUP_INDEX_PREFIX]]widget/$[[SETUP_DOC_TYPE]]/cji1sc28go5i051pl1i0
|
|||
"formula": "a",
|
||||
"items": [
|
||||
{
|
||||
"field": "*",
|
||||
"field": "id",
|
||||
"name": "a",
|
||||
"statistic": "count"
|
||||
}
|
||||
|
|
@ -51,7 +51,7 @@ POST $[[SETUP_INDEX_PREFIX]]widget/$[[SETUP_DOC_TYPE]]/cji1ttq8go5i051pl1t2
|
|||
"formula": "a",
|
||||
"items": [
|
||||
{
|
||||
"field": "*",
|
||||
"field": "id",
|
||||
"name": "a",
|
||||
"statistic": "count"
|
||||
}
|
||||
|
|
@ -94,7 +94,7 @@ POST $[[SETUP_INDEX_PREFIX]]widget/$[[SETUP_DOC_TYPE]]/cji1ttq8go5i051pl1t1
|
|||
],
|
||||
"items": [
|
||||
{
|
||||
"field": "*",
|
||||
"field": "id",
|
||||
"name": "a",
|
||||
"statistic": "count"
|
||||
}
|
||||
|
|
@ -137,7 +137,7 @@ POST $[[SETUP_INDEX_PREFIX]]widget/$[[SETUP_DOC_TYPE]]/cji1ttq8go5i051pl1t0
|
|||
],
|
||||
"items": [
|
||||
{
|
||||
"field": "*",
|
||||
"field": "id",
|
||||
"name": "a",
|
||||
"statistic": "count"
|
||||
}
|
||||
|
|
@ -3566,4 +3566,579 @@ POST $[[SETUP_INDEX_PREFIX]]layout/$[[SETUP_DOC_TYPE]]/cicmhbt3q95ich72lrvg
|
|||
},
|
||||
"type": "workspace",
|
||||
"is_fixed": true
|
||||
}
|
||||
|
||||
#shard level
|
||||
PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH9
|
||||
{
|
||||
"id": "bD2jH5QB7KvGccywNCH9",
|
||||
"name": "Indexing Rate",
|
||||
"key": "indexing_rate",
|
||||
"level": "shard",
|
||||
"formula": "bD2jH5QB7KvGccywNCH9/{{.bucket_size_in_second}}",
|
||||
"items": [
|
||||
{
|
||||
"name": "bD2jH5QB7KvGccywNCH9",
|
||||
"field": "payload.elasticsearch.shard_stats.indexing.index_total",
|
||||
"statistic": "rate"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "doc/s",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH1
|
||||
{
|
||||
"id": "bD2jH5QB7KvGccywNCH1",
|
||||
"name": "Shard Storage",
|
||||
"key": "shard_storage",
|
||||
"level": "shard",
|
||||
"formula": "bD2jH5QB7KvGccywNCH1",
|
||||
"items": [
|
||||
{
|
||||
"name": "bD2jH5QB7KvGccywNCH1",
|
||||
"field": "payload.elasticsearch.shard_stats.store.size_in_bytes",
|
||||
"statistic": "max"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "bytes",
|
||||
"unit": "",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH5
|
||||
{
|
||||
"id": "bD2jH5QB7KvGccywNCH5",
|
||||
"name": "Document Count",
|
||||
"key": "doc_count",
|
||||
"level": "shard",
|
||||
"formula": "bD2jH5QB7KvGccywNCH5",
|
||||
"items": [
|
||||
{
|
||||
"name": "bD2jH5QB7KvGccywNCH5",
|
||||
"field": "payload.elasticsearch.shard_stats.docs.count",
|
||||
"statistic": "max"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "number",
|
||||
"unit": "",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH2
|
||||
{
|
||||
"id": "bD2jH5QB7KvGccywNCH2",
|
||||
"name": "Search Rate",
|
||||
"key": "search_rate",
|
||||
"level": "shard",
|
||||
"formula": "bD2jH5QB7KvGccywNCH2/{{.bucket_size_in_second}}",
|
||||
"items": [
|
||||
{
|
||||
"name": "bD2jH5QB7KvGccywNCH2",
|
||||
"field": "payload.elasticsearch.shard_stats.search.query_total",
|
||||
"statistic": "rate"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "doc/s",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH3
|
||||
{
|
||||
"id": "bD2jH5QB7KvGccywNCH3",
|
||||
"name": "Indexing Latency",
|
||||
"key": "indexing_latency",
|
||||
"level": "shard",
|
||||
"formula": "bD2jH5QB7KvGccywNCx3/bD2jH5QB7KvGccywNCH3",
|
||||
"items": [
|
||||
{
|
||||
"name": "bD2jH5QB7KvGccywNCx3",
|
||||
"field": "payload.elasticsearch.shard_stats.indexing.index_total",
|
||||
"statistic": "rate"
|
||||
},
|
||||
{
|
||||
"name": "bD2jH5QB7KvGccywNCH3",
|
||||
"field": "payload.elasticsearch.shard_stats.indexing.index_time_in_millis",
|
||||
"statistic": "rate"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "ms",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH4
|
||||
{
|
||||
"id": "bD2jH5QB7KvGccywNCH4",
|
||||
"name": "Search Latency",
|
||||
"key": "search_latency",
|
||||
"level": "shard",
|
||||
"formula": "bD2jH5QB7KvGccywNCx4/bD2jH5QB7KvGccywNCH4",
|
||||
"items": [
|
||||
{
|
||||
"name": "bD2jH5QB7KvGccywNCH4",
|
||||
"field": "payload.elasticsearch.shard_stats.search.query_total",
|
||||
"statistic": "rate"
|
||||
},
|
||||
{
|
||||
"name": "bD2jH5QB7KvGccywNCx4",
|
||||
"field": "payload.elasticsearch.shard_stats.search.query_time_in_millis",
|
||||
"statistic": "rate"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "ms",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH6
|
||||
{
|
||||
"id": "bD2jH5QB7KvGccywNCH6",
|
||||
"name": "Segment Count",
|
||||
"key": "segment_count",
|
||||
"level": "shard",
|
||||
"formula": "bD2jH5QB7KvGccywNCH6",
|
||||
"items": [
|
||||
{
|
||||
"name": "bD2jH5QB7KvGccywNCH6",
|
||||
"field": "payload.elasticsearch.shard_stats.segments.count",
|
||||
"statistic": "max"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "number",
|
||||
"unit": "",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH7
|
||||
{
|
||||
"id": "bD2jH5QB7KvGccywNCH7",
|
||||
"name": "Segment memory",
|
||||
"key": "segment_memory",
|
||||
"level": "shard",
|
||||
"formula": "bD2jH5QB7KvGccywNCH7",
|
||||
"items": [
|
||||
{
|
||||
"name": "bD2jH5QB7KvGccywNCH7",
|
||||
"field": "payload.elasticsearch.shard_stats.segments.memory_in_bytes",
|
||||
"statistic": "max"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "number",
|
||||
"unit": "",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
#indices level
|
||||
PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH9
|
||||
{
|
||||
"id": "aD2jH5QB7KvGccywNCH9",
|
||||
"name": "Indexing Rate",
|
||||
"key": "indexing_rate",
|
||||
"level": "indices",
|
||||
"formula": "aD2jH5QB7KvGccywNCH9/{{.bucket_size_in_second}}",
|
||||
"items": [
|
||||
{
|
||||
"name": "aD2jH5QB7KvGccywNCH9",
|
||||
"field": "payload.elasticsearch.index_stats.primaries.indexing.index_total",
|
||||
"statistic": "rate"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "doc/s",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH1
|
||||
{
|
||||
"id": "aD2jH5QB7KvGccywNCH1",
|
||||
"name": "Index Storage",
|
||||
"key": "index_storage",
|
||||
"level": "indices",
|
||||
"formula": "aD2jH5QB7KvGccywNCH1",
|
||||
"items": [
|
||||
{
|
||||
"name": "aD2jH5QB7KvGccywNCH1",
|
||||
"field": "payload.elasticsearch.index_stats.total.store.size_in_bytes",
|
||||
"statistic": "max"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "bytes",
|
||||
"unit": "",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH5
|
||||
{
|
||||
"id": "aD2jH5QB7KvGccywNCH5",
|
||||
"name": "Document Count",
|
||||
"key": "doc_count",
|
||||
"level": "indices",
|
||||
"formula": "aD2jH5QB7KvGccywNCH5",
|
||||
"items": [
|
||||
{
|
||||
"name": "aD2jH5QB7KvGccywNCH5",
|
||||
"field": "payload.elasticsearch.index_stats.total.docs.count",
|
||||
"statistic": "max"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "number",
|
||||
"unit": "",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH2
|
||||
{
|
||||
"id": "aD2jH5QB7KvGccywNCH2",
|
||||
"name": "Search Rate",
|
||||
"key": "search_rate",
|
||||
"level": "indices",
|
||||
"formula": "aD2jH5QB7KvGccywNCH2/{{.bucket_size_in_second}}",
|
||||
"items": [
|
||||
{
|
||||
"name": "aD2jH5QB7KvGccywNCH2",
|
||||
"field": "payload.elasticsearch.index_stats.total.search.query_total",
|
||||
"statistic": "rate"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "doc/s",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH3
|
||||
{
|
||||
"id": "aD2jH5QB7KvGccywNCH3",
|
||||
"name": "Indexing Latency",
|
||||
"key": "indexing_latency",
|
||||
"level": "indices",
|
||||
"formula": "aD2jH5QB7KvGccywNCx3/aD2jH5QB7KvGccywNCH3",
|
||||
"items": [
|
||||
{
|
||||
"name": "aD2jH5QB7KvGccywNCH3",
|
||||
"field": "payload.elasticsearch.index_stats.primaries.indexing.index_total",
|
||||
"statistic": "rate"
|
||||
},
|
||||
{
|
||||
"name": "aD2jH5QB7KvGccywNCx3",
|
||||
"field": "payload.elasticsearch.index_stats.primaries.indexing.index_time_in_millis",
|
||||
"statistic": "rate"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "ms",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH4
|
||||
{
|
||||
"id": "aD2jH5QB7KvGccywNCH4",
|
||||
"name": "Search Latency",
|
||||
"key": "search_latency",
|
||||
"level": "indices",
|
||||
"formula": "aD2jH5QB7KvGccywNCx4/aD2jH5QB7KvGccywNCH4",
|
||||
"items": [
|
||||
{
|
||||
"name": "aD2jH5QB7KvGccywNCH4",
|
||||
"field": "payload.elasticsearch.index_stats.total.search.query_total",
|
||||
"statistic": "rate"
|
||||
},
|
||||
{
|
||||
"name": "aD2jH5QB7KvGccywNCx4",
|
||||
"field": "payload.elasticsearch.index_stats.total.search.query_time_in_millis",
|
||||
"statistic": "rate"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "ms",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH6
|
||||
{
|
||||
"id": "aD2jH5QB7KvGccywNCH6",
|
||||
"name": "Segment Count",
|
||||
"key": "segment_count",
|
||||
"level": "indices",
|
||||
"formula": "aD2jH5QB7KvGccywNCH6",
|
||||
"items": [
|
||||
{
|
||||
"name": "aD2jH5QB7KvGccywNCH6",
|
||||
"field": "payload.elasticsearch.index_stats.total.segments.count",
|
||||
"statistic": "max"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "number",
|
||||
"unit": "",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH7
|
||||
{
|
||||
"id": "aD2jH5QB7KvGccywNCH7",
|
||||
"name": "Segment memory",
|
||||
"key": "segment_memory",
|
||||
"level": "indices",
|
||||
"formula": "aD2jH5QB7KvGccywNCH7",
|
||||
"items": [
|
||||
{
|
||||
"name": "aD2jH5QB7KvGccywNCH7",
|
||||
"field": "payload.elasticsearch.index_stats.total.segments.memory_in_bytes",
|
||||
"statistic": "max"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "bytes",
|
||||
"unit": "",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
#node level
|
||||
PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH9
|
||||
{
|
||||
"id": "jD2jH5QB7KvGccywNCH9",
|
||||
"name": "Indexing Rate",
|
||||
"key": "indexing_rate",
|
||||
"level": "node",
|
||||
"formula": "jD2jH5QB7KvGccywH9/{{.bucket_size_in_second}}",
|
||||
"items": [
|
||||
{
|
||||
"name": "jD2jH5QB7KvGccywH9",
|
||||
"field": "payload.elasticsearch.node_stats.indices.indexing.index_total",
|
||||
"statistic": "derivative"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "doc/s",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
|
||||
PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH4
|
||||
{
|
||||
"id": "jD2jH5QB7KvGccywNCH4",
|
||||
"name": "Process CPU Usage",
|
||||
"key": "process_cpu_used",
|
||||
"level": "node",
|
||||
"formula": "jD2jH5QB7KvGccywNCH4",
|
||||
"items": [
|
||||
{
|
||||
"name": "jD2jH5QB7KvGccywNCH4",
|
||||
"field": "payload.elasticsearch.node_stats.process.cpu.percent",
|
||||
"statistic": "avg"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "",
|
||||
"unit": "%",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH3
|
||||
{
|
||||
"id": "jD2jH5QB7KvGccywNCH3",
|
||||
"name": "JVM Heap Usage",
|
||||
"key": "jvm_heap_used",
|
||||
"level": "node",
|
||||
"formula": "jD2jH5QB7KvGccywNCH3",
|
||||
"items": [
|
||||
{
|
||||
"name": "jD2jH5QB7KvGccywNCH3",
|
||||
"field": "payload.elasticsearch.node_stats.jvm.mem.heap_used_in_bytes",
|
||||
"statistic": "max"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "bytes",
|
||||
"unit": "",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH1
|
||||
{
|
||||
"id": "jD2jH5QB7KvGccywNCH1",
|
||||
"name": "Indexing Latency",
|
||||
"key": "indexing_latency",
|
||||
"level": "node",
|
||||
"formula": "jD2jH5QB7KvGccywNCx1/jD2jH5QB7KvGccywNCH1",
|
||||
"items": [
|
||||
{
|
||||
"name": "jD2jH5QB7KvGccywNCH1",
|
||||
"field": "payload.elasticsearch.node_stats.indices.indexing.index_total",
|
||||
"statistic": "rate"
|
||||
},
|
||||
{
|
||||
"name": "jD2jH5QB7KvGccywNCx1",
|
||||
"field": "payload.elasticsearch.node_stats.indices.indexing.index_time_in_millis",
|
||||
"statistic": "rate"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "ms",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH0
|
||||
{
|
||||
"id": "jD2jH5QB7KvGccywNCH0",
|
||||
"name": "Search Rate",
|
||||
"key": "search_rate",
|
||||
"level": "node",
|
||||
"formula": "jD2jH5QB7KvGccywH0/{{.bucket_size_in_second}}",
|
||||
"items": [
|
||||
{
|
||||
"name": "jD2jH5QB7KvGccywH0",
|
||||
"field": "payload.elasticsearch.node_stats.indices.search.query_total",
|
||||
"statistic": "derivative"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "query/s",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH9
|
||||
{
|
||||
"id": "jD2jH5QB7KvGccywNCH9",
|
||||
"name": "Indexing Rate",
|
||||
"key": "indexing_rate",
|
||||
"level": "node",
|
||||
"formula": "jD2jH5QB7KvGccywH9/{{.bucket_size_in_second}}",
|
||||
"items": [
|
||||
{
|
||||
"name": "jD2jH5QB7KvGccywH9",
|
||||
"field": "payload.elasticsearch.node_stats.indices.indexing.index_total",
|
||||
"statistic": "rate"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "doc/s",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH5
|
||||
{
|
||||
"id": "jD2jH5QB7KvGccywNCH5",
|
||||
"name": "Search Latency",
|
||||
"key": "search_latency",
|
||||
"level": "node",
|
||||
"formula": "jD2jH5QB7KvGccywNCx5/jD2jH5QB7KvGccywNCH5",
|
||||
"items": [
|
||||
{
|
||||
"name": "jD2jH5QB7KvGccywNCH5",
|
||||
"field": "payload.elasticsearch.node_stats.indices.search.query_total",
|
||||
"statistic": "rate"
|
||||
},
|
||||
{
|
||||
"name": "jD2jH5QB7KvGccywNCx5",
|
||||
"field": "payload.elasticsearch.node_stats.indices.search.query_time_in_millis",
|
||||
"statistic": "rate"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "ms",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH6
|
||||
{
|
||||
"id": "jD2jH5QB7KvGccywNCH6",
|
||||
"name": "Indices Storage",
|
||||
"key": "indices_storage",
|
||||
"level": "node",
|
||||
"formula": "jD2jH5QB7KvGccywNCH6",
|
||||
"items": [
|
||||
{
|
||||
"name": "jD2jH5QB7KvGccywNCH6",
|
||||
"field": "payload.elasticsearch.node_stats.indices.store.size_in_bytes",
|
||||
"statistic": "max"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "bytes",
|
||||
"unit": "",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH7
|
||||
{
|
||||
"id": "jD2jH5QB7KvGccywNCH7",
|
||||
"name": "Document Count",
|
||||
"key": "doc_count",
|
||||
"level": "node",
|
||||
"formula": "jD2jH5QB7KvGccywNCH7",
|
||||
"items": [
|
||||
{
|
||||
"name": "jD2jH5QB7KvGccywNCH7",
|
||||
"field": "payload.elasticsearch.node_stats.indices.docs.count",
|
||||
"statistic": "max"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "number",
|
||||
"unit": "",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
|
@ -14,7 +14,9 @@ Information about release notes of INFINI Console is provided here.
|
|||
### Features
|
||||
- Add allocation to activities if is cluster health change and changed to red.
|
||||
- Add index metrics for segment memory (norms, points, version map, fixed bit set).
|
||||
|
||||
- Support querying top N metrics in the Insight Data Query API
|
||||
- Add insight metric CURD API for managing custom metrics
|
||||
- Add built-in metrics templates for common use cases
|
||||
### Bug fix
|
||||
- Fixed query thread pool metrics when cluster uuid is empty
|
||||
- Fixed unit tests
|
||||
|
|
|
|||
1
main.go
1
main.go
|
|
@ -158,6 +158,7 @@ func main() {
|
|||
orm.RegisterSchemaWithIndexName(api3.RemoteConfig{}, "configs")
|
||||
orm.RegisterSchemaWithIndexName(model.AuditLog{}, "audit-logs")
|
||||
orm.RegisterSchemaWithIndexName(host.HostInfo{}, "host")
|
||||
orm.RegisterSchemaWithIndexName(insight.MetricBase{}, "metric")
|
||||
|
||||
module.Start()
|
||||
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ package insight
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"infini.sh/framework/core/orm"
|
||||
"infini.sh/framework/core/util"
|
||||
"regexp"
|
||||
)
|
||||
|
|
@ -43,11 +44,35 @@ type Metric struct {
|
|||
Sort []GroupSort `json:"sort,omitempty"`
|
||||
ClusterId string `json:"cluster_id,omitempty"`
|
||||
Formula string `json:"formula,omitempty"`
|
||||
//array of formula for new version
|
||||
Formulas []string `json:"formulas,omitempty"`
|
||||
Items []MetricItem `json:"items"`
|
||||
FormatType string `json:"format_type,omitempty"`
|
||||
FormatType string `json:"format,omitempty"`
|
||||
TimeFilter interface{} `json:"time_filter,omitempty"`
|
||||
TimeBeforeGroup bool `json:"time_before_group,omitempty"`
|
||||
BucketLabel *BucketLabel `json:"bucket_label,omitempty"`
|
||||
// number of buckets to return, used for aggregation auto_date_histogram when bucket size equals 'auto'
|
||||
Buckets uint `json:"buckets,omitempty"`
|
||||
Unit string `json:"unit,omitempty"`
|
||||
}
|
||||
|
||||
type MetricBase struct {
|
||||
orm.ORMObjectBase
|
||||
//display name of the metric
|
||||
Name string `json:"name"`
|
||||
//metric identifier
|
||||
Key string `json:"key"`
|
||||
//optional values : "node", "indices", "shard"
|
||||
Level string `json:"level"`
|
||||
//metric calculation formula
|
||||
Formula string `json:"formula,omitempty"`
|
||||
Items []MetricItem `json:"items"`
|
||||
FormatType string `json:"format,omitempty"`
|
||||
Unit string `json:"unit,omitempty"`
|
||||
//determine if this metric is built-in
|
||||
Builtin bool `json:"builtin"`
|
||||
//array of supported calculation statistic, eg: "avg", "sum", "min", "max"
|
||||
Statistics []string `json:"statistics,omitempty"`
|
||||
}
|
||||
|
||||
type GroupSort struct {
|
||||
|
|
@ -105,12 +130,8 @@ func (m *Metric) ValidateSortKey() error {
|
|||
if !util.StringInArray([]string{"desc", "asc"}, sortItem.Direction){
|
||||
return fmt.Errorf("unknown sort direction [%s]", sortItem.Direction)
|
||||
}
|
||||
if v, ok := mm[sortItem.Key]; !ok && !util.StringInArray([]string{"_key", "_count"}, sortItem.Key){
|
||||
if _, ok := mm[sortItem.Key]; !ok && !util.StringInArray([]string{"_key", "_count"}, sortItem.Key){
|
||||
return fmt.Errorf("unknown sort key [%s]", sortItem.Key)
|
||||
}else{
|
||||
if v != nil && v.Statistic == "derivative" {
|
||||
return fmt.Errorf("can not sort by pipeline agg [%s]", v.Statistic)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -56,4 +56,7 @@ func InitAPI() {
|
|||
api.HandleAPIMethod(api.POST, "/elasticsearch/:id/map_label/_render", insight.renderMapLabelTemplate)
|
||||
api.HandleAPIMethod(api.GET, "/insight/widget/:widget_id", insight.getWidget)
|
||||
api.HandleAPIMethod(api.POST, "/insight/widget", insight.RequireLogin(insight.createWidget))
|
||||
api.HandleAPIMethod(api.POST, "/insight/metric", insight.createMetric)
|
||||
api.HandleAPIMethod(api.PUT, "/insight/metric/:metric_id", insight.updateMetric)
|
||||
api.HandleAPIMethod(api.DELETE, "/insight/metric/:metric_id", insight.deleteMetric)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -248,8 +248,8 @@ func getMetricData(metric *insight.Metric) (interface{}, error) {
|
|||
return nil, err
|
||||
}
|
||||
esClient := elastic.GetClient(metric.ClusterId)
|
||||
//log.Error(string(util.MustToJSONBytes(query)))
|
||||
searchRes, err := esClient.SearchWithRawQueryDSL(metric.IndexPattern, util.MustToJSONBytes(query))
|
||||
queryDSL := util.MustToJSONBytes(query)
|
||||
searchRes, err := esClient.SearchWithRawQueryDSL(metric.IndexPattern, queryDSL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -266,83 +266,101 @@ func getMetricData(metric *insight.Metric) (interface{}, error) {
|
|||
}
|
||||
}
|
||||
timeBeforeGroup := metric.AutoTimeBeforeGroup()
|
||||
metricData := CollectMetricData(agg, timeBeforeGroup)
|
||||
metricData, interval := CollectMetricData(agg, timeBeforeGroup)
|
||||
formula := strings.TrimSpace(metric.Formula)
|
||||
//support older versions for a single formula.
|
||||
if metric.Formula != "" && len(metric.Formulas) == 0 {
|
||||
metric.Formulas = []string{metric.Formula}
|
||||
}
|
||||
|
||||
var targetMetricData []insight.MetricData
|
||||
formula := strings.TrimSpace(metric.Formula)
|
||||
if len(metric.Items) == 1 && formula == "" {
|
||||
if len(metric.Items) == 1 && len(metric.Formulas) == 0 {
|
||||
targetMetricData = metricData
|
||||
} else {
|
||||
tpl, err := template.New("insight_formula").Parse(formula)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
msgBuffer := &bytes.Buffer{}
|
||||
params := map[string]interface{}{}
|
||||
if metric.BucketSize != "" {
|
||||
du, err := util.ParseDuration(metric.BucketSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
bucketSize := metric.BucketSize
|
||||
if metric.BucketSize == "auto" && interval != "" {
|
||||
bucketSize = interval
|
||||
}
|
||||
if interval != "" || bucketSize != "auto" {
|
||||
du, err := util.ParseDuration(bucketSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
params["bucket_size_in_second"] = du.Seconds()
|
||||
}
|
||||
params["bucket_size_in_second"] = du.Seconds()
|
||||
}
|
||||
err = tpl.Execute(msgBuffer, params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
formula = msgBuffer.String()
|
||||
for _, md := range metricData {
|
||||
targetData := insight.MetricData{
|
||||
Groups: md.Groups,
|
||||
Data: map[string][]insight.MetricDataItem{},
|
||||
}
|
||||
expression, err := govaluate.NewEvaluableExpression(formula)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dataLength := 0
|
||||
for _, v := range md.Data {
|
||||
dataLength = len(v)
|
||||
break
|
||||
}
|
||||
DataLoop:
|
||||
for i := 0; i < dataLength; i++ {
|
||||
parameters := map[string]interface{}{}
|
||||
var timestamp interface{}
|
||||
hasValidData := false
|
||||
for k, v := range md.Data {
|
||||
if len(k) == 20 {
|
||||
continue
|
||||
}
|
||||
if len(v) < dataLength {
|
||||
continue
|
||||
}
|
||||
if _, ok := v[i].Value.(float64); !ok {
|
||||
continue DataLoop
|
||||
}
|
||||
hasValidData = true
|
||||
parameters[k] = v[i].Value
|
||||
timestamp = v[i].Timestamp
|
||||
}
|
||||
//todo return error?
|
||||
if !hasValidData {
|
||||
continue
|
||||
}
|
||||
result, err := expression.Evaluate(parameters)
|
||||
retMetricDataItem := insight.MetricDataItem{}
|
||||
for _, formula = range metric.Formulas {
|
||||
tpl, err := template.New("insight_formula").Parse(formula)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r, ok := result.(float64); ok {
|
||||
if math.IsNaN(r) || math.IsInf(r, 0) {
|
||||
//if !isFilterNaN {
|
||||
// targetData.Data["result"] = append(targetData.Data["result"], []interface{}{timestamp, math.NaN()})
|
||||
//}
|
||||
msgBuffer := &bytes.Buffer{}
|
||||
err = tpl.Execute(msgBuffer, params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resolvedFormula := msgBuffer.String()
|
||||
expression, err := govaluate.NewEvaluableExpression(resolvedFormula)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dataLength := 0
|
||||
for _, v := range md.Data {
|
||||
dataLength = len(v)
|
||||
break
|
||||
}
|
||||
DataLoop:
|
||||
for i := 0; i < dataLength; i++ {
|
||||
parameters := map[string]interface{}{}
|
||||
var timestamp interface{}
|
||||
hasValidData := false
|
||||
for k, v := range md.Data {
|
||||
if _, ok := v[i].Value.(float64); !ok {
|
||||
continue DataLoop
|
||||
}
|
||||
hasValidData = true
|
||||
parameters[k] = v[i].Value
|
||||
timestamp = v[i].Timestamp
|
||||
}
|
||||
//todo return error?
|
||||
if !hasValidData {
|
||||
continue
|
||||
}
|
||||
result, err := expression.Evaluate(parameters)
|
||||
if err != nil {
|
||||
log.Debugf("evaluate formula error: %v", err)
|
||||
continue
|
||||
}
|
||||
if r, ok := result.(float64); ok {
|
||||
if math.IsNaN(r) || math.IsInf(r, 0) {
|
||||
//if !isFilterNaN {
|
||||
// targetData.Data["result"] = append(targetData.Data["result"], []interface{}{timestamp, math.NaN()})
|
||||
//}
|
||||
continue
|
||||
}
|
||||
}
|
||||
retMetricDataItem.Timestamp = timestamp
|
||||
if len(metric.Formulas) <= 1 && metric.Formula != ""{
|
||||
//support older versions by returning the result for a single formula.
|
||||
retMetricDataItem.Value = result
|
||||
} else {
|
||||
if v, ok := retMetricDataItem.Value.(map[string]interface{}); ok {
|
||||
v[formula] = result
|
||||
}else{
|
||||
retMetricDataItem.Value = map[string]interface{}{formula: result}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
targetData.Data["result"] = append(targetData.Data["result"], insight.MetricDataItem{Timestamp: timestamp, Value: result})
|
||||
}
|
||||
targetData.Data["result"] = append(targetData.Data["result"], retMetricDataItem)
|
||||
targetMetricData = append(targetMetricData, targetData)
|
||||
}
|
||||
}
|
||||
|
|
@ -356,7 +374,10 @@ func getMetricData(metric *insight.Metric) (interface{}, error) {
|
|||
}
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
return util.MapStr{
|
||||
"data": result,
|
||||
"request": string(queryDSL),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func getMetadataByIndexPattern(clusterID, indexPattern, timeField string, filter interface{}, fieldsFormat map[string]string) (interface{}, error) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,166 @@
|
|||
// Copyright (C) INFINI Labs & INFINI LIMITED.
|
||||
//
|
||||
// The INFINI Console is offered under the GNU Affero General Public License v3.0
|
||||
// and as commercial software.
|
||||
//
|
||||
// For commercial licensing, contact us at:
|
||||
// - Website: infinilabs.com
|
||||
// - Email: hello@infini.ltd
|
||||
//
|
||||
// Open Source licensed under AGPL V3:
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
/* Copyright © INFINI Ltd. All rights reserved.
|
||||
* web: https://infinilabs.com
|
||||
* mail: hello#infini.ltd */
|
||||
|
||||
package insight
|
||||
|
||||
import (
|
||||
"errors"
|
||||
log "github.com/cihub/seelog"
|
||||
"infini.sh/console/model/insight"
|
||||
httprouter "infini.sh/framework/core/api/router"
|
||||
"infini.sh/framework/core/orm"
|
||||
"infini.sh/framework/core/util"
|
||||
"infini.sh/framework/modules/elastic"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func (h *InsightAPI) createMetric(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
|
||||
var obj = &insight.MetricBase{}
|
||||
err := h.DecodeJSON(req, obj)
|
||||
if err != nil {
|
||||
h.WriteError(w, err.Error(), http.StatusInternalServerError)
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
err = orm.Create(nil, obj)
|
||||
if err != nil {
|
||||
h.WriteError(w, err.Error(), http.StatusInternalServerError)
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
h.WriteJSON(w, util.MapStr{
|
||||
"_id": obj.ID,
|
||||
"result": "created",
|
||||
}, 200)
|
||||
|
||||
}
|
||||
|
||||
func (h *InsightAPI) getMetric(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
|
||||
id := ps.MustGetParameter("metric_id")
|
||||
|
||||
obj := insight.MetricBase{}
|
||||
obj.ID = id
|
||||
|
||||
_, err := orm.Get(&obj)
|
||||
if err != nil {
|
||||
if errors.Is(err, elastic.ErrNotFound) {
|
||||
h.WriteJSON(w, util.MapStr{
|
||||
"_id": id,
|
||||
"found": false,
|
||||
}, http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
h.WriteError(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
h.WriteJSON(w, util.MapStr{
|
||||
"found": true,
|
||||
"_id": id,
|
||||
"_source": obj,
|
||||
}, 200)
|
||||
}
|
||||
|
||||
func (h *InsightAPI) updateMetric(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
|
||||
id := ps.MustGetParameter("metric_id")
|
||||
obj := insight.MetricBase{}
|
||||
|
||||
obj.ID = id
|
||||
_, err := orm.Get(&obj)
|
||||
if err != nil {
|
||||
if errors.Is(err, elastic.ErrNotFound) {
|
||||
h.WriteJSON(w, util.MapStr{
|
||||
"_id": id,
|
||||
"found": false,
|
||||
}, http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
h.WriteError(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
id = obj.ID
|
||||
create := obj.Created
|
||||
obj = insight.MetricBase{}
|
||||
err = h.DecodeJSON(req, &obj)
|
||||
if err != nil {
|
||||
h.WriteError(w, err.Error(), http.StatusInternalServerError)
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
//protect
|
||||
obj.ID = id
|
||||
obj.Created = create
|
||||
err = orm.Update(nil, &obj)
|
||||
if err != nil {
|
||||
h.WriteError(w, err.Error(), http.StatusInternalServerError)
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
h.WriteJSON(w, util.MapStr{
|
||||
"_id": obj.ID,
|
||||
"result": "updated",
|
||||
}, 200)
|
||||
}
|
||||
|
||||
func (h *InsightAPI) deleteMetric(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
|
||||
id := ps.MustGetParameter("metric_id")
|
||||
|
||||
obj := insight.MetricBase{}
|
||||
obj.ID = id
|
||||
|
||||
_, err := orm.Get(&obj)
|
||||
if err != nil {
|
||||
if errors.Is(err, elastic.ErrNotFound) {
|
||||
h.WriteJSON(w, util.MapStr{
|
||||
"_id": id,
|
||||
"found": false,
|
||||
}, http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
h.WriteError(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if obj.Builtin {
|
||||
h.WriteError(w, "cannot delete builtin metrics", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
err = orm.Delete(nil, &obj)
|
||||
if err != nil {
|
||||
h.WriteError(w, err.Error(), http.StatusInternalServerError)
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
h.WriteJSON(w, util.MapStr{
|
||||
"_id": obj.ID,
|
||||
"result": "deleted",
|
||||
}, 200)
|
||||
}
|
||||
|
|
@ -72,7 +72,7 @@ func generateAgg(metricItem *insight.MetricItem, timeField string) map[string]in
|
|||
"includes": []string{field},
|
||||
}
|
||||
aggValue["sort"] = []util.MapStr{
|
||||
util.MapStr{
|
||||
{
|
||||
timeField: util.MapStr{
|
||||
"order": "desc",
|
||||
},
|
||||
|
|
@ -114,23 +114,46 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) {
|
|||
return nil, err
|
||||
}
|
||||
}
|
||||
verInfo := elastic.GetClient(metric.ClusterId).GetVersion()
|
||||
var (
|
||||
useDateHistogram = false
|
||||
dateHistogramAgg util.MapStr
|
||||
dateHistogramAggName string
|
||||
)
|
||||
if metric.BucketSize != "" && metric.TimeField != "" {
|
||||
useDateHistogram = true
|
||||
if metric.BucketSize == "auto" {
|
||||
dateHistogramAggName = "auto_date_histogram"
|
||||
buckets := metric.Buckets
|
||||
if buckets == 0 {
|
||||
buckets = 2
|
||||
}
|
||||
dateHistogramAgg = util.MapStr{
|
||||
"field": metric.TimeField,
|
||||
"buckets": buckets,
|
||||
}
|
||||
}else{
|
||||
dateHistogramAggName = "date_histogram"
|
||||
verInfo := elastic.GetClient(metric.ClusterId).GetVersion()
|
||||
|
||||
if verInfo.Number == "" {
|
||||
panic("invalid version")
|
||||
if verInfo.Number == "" {
|
||||
panic("invalid version")
|
||||
}
|
||||
|
||||
intervalField, err := elastic.GetDateHistogramIntervalField(verInfo.Distribution, verInfo.Number, metric.BucketSize)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get interval field error: %w", err)
|
||||
}
|
||||
dateHistogramAgg = util.MapStr{
|
||||
"field": metric.TimeField,
|
||||
intervalField: metric.BucketSize,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
intervalField, err := elastic.GetDateHistogramIntervalField(verInfo.Distribution, verInfo.Number, metric.BucketSize)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get interval field error: %w", err)
|
||||
}
|
||||
if metric.BucketSize != "" && !timeBeforeGroup {
|
||||
if useDateHistogram && !timeBeforeGroup {
|
||||
basicAggs = util.MapStr{
|
||||
"time_buckets": util.MapStr{
|
||||
"date_histogram": util.MapStr{
|
||||
"field": metric.TimeField,
|
||||
intervalField: metric.BucketSize,
|
||||
},
|
||||
dateHistogramAggName: dateHistogramAgg,
|
||||
"aggs": basicAggs,
|
||||
},
|
||||
}
|
||||
|
|
@ -138,7 +161,7 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) {
|
|||
|
||||
var rootAggs util.MapStr
|
||||
groups := metric.Groups
|
||||
err = metric.ValidateSortKey()
|
||||
err := metric.ValidateSortKey()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -156,12 +179,43 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) {
|
|||
"field": groups[i].Field,
|
||||
"size": limit,
|
||||
}
|
||||
if i == grpLength-1 && len(metric.Sort) > 0 {
|
||||
var termsOrder []interface{}
|
||||
for _, sortItem := range metric.Sort {
|
||||
termsOrder = append(termsOrder, util.MapStr{sortItem.Key: sortItem.Direction})
|
||||
if i == grpLength - 1 && len(metric.Sort) > 0 {
|
||||
//use bucket sort instead of terms order when time after group
|
||||
if !timeBeforeGroup && len(metric.Sort) > 0 {
|
||||
basicAggs["sort_field"] = util.MapStr{
|
||||
"max_bucket": util.MapStr{
|
||||
"buckets_path": fmt.Sprintf("time_buckets>%s", metric.Sort[0].Key),
|
||||
},
|
||||
}
|
||||
//using 65536 as a workaround for the terms aggregation limit; the actual limit is enforced in the bucket sort step
|
||||
termsCfg["size"] = 65536
|
||||
basicAggs["bucket_sorter"] = util.MapStr{
|
||||
"bucket_sort": util.MapStr{
|
||||
"size": limit,
|
||||
"sort": []util.MapStr{
|
||||
{"sort_field": util.MapStr{"order": metric.Sort[0].Direction}},
|
||||
},
|
||||
},
|
||||
}
|
||||
}else{
|
||||
var termsOrder []interface{}
|
||||
percentAggs := []string{"p99", "p95", "p90", "p80", "p50"}
|
||||
for _, sortItem := range metric.Sort {
|
||||
var percent string
|
||||
for _, item := range metric.Items {
|
||||
lowerCaseStatistic := strings.ToLower(item.Statistic)
|
||||
if item.Name == sortItem.Key && util.StringInArray(percentAggs, lowerCaseStatistic) {
|
||||
percent = lowerCaseStatistic[1:]
|
||||
}
|
||||
}
|
||||
sortKey := sortItem.Key
|
||||
if percent != "" {
|
||||
sortKey = fmt.Sprintf("%s[%s]", sortItem.Key, percent)
|
||||
}
|
||||
termsOrder = append(termsOrder, util.MapStr{sortKey: sortItem.Direction})
|
||||
}
|
||||
termsCfg["order"] = termsOrder
|
||||
}
|
||||
termsCfg["order"] = termsOrder
|
||||
}
|
||||
groupAgg := util.MapStr{
|
||||
"terms": termsCfg,
|
||||
|
|
@ -176,32 +230,26 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) {
|
|||
}
|
||||
lastGroupAgg = groupAgg
|
||||
}
|
||||
if metric.BucketSize == "" || (metric.BucketSize != "" && !timeBeforeGroup) {
|
||||
rootAggs = util.MapStr{
|
||||
util.GetUUID(): lastGroupAgg,
|
||||
}
|
||||
} else {
|
||||
if useDateHistogram && timeBeforeGroup {
|
||||
rootAggs = util.MapStr{
|
||||
"time_buckets": util.MapStr{
|
||||
"date_histogram": util.MapStr{
|
||||
"field": metric.TimeField,
|
||||
intervalField: metric.BucketSize,
|
||||
},
|
||||
dateHistogramAggName: dateHistogramAgg,
|
||||
"aggs": util.MapStr{
|
||||
util.GetUUID(): lastGroupAgg,
|
||||
},
|
||||
},
|
||||
}
|
||||
} else {
|
||||
rootAggs = util.MapStr{
|
||||
util.GetUUID(): lastGroupAgg,
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
if metric.BucketSize != "" && timeBeforeGroup {
|
||||
basicAggs = util.MapStr{
|
||||
"time_buckets": util.MapStr{
|
||||
"date_histogram": util.MapStr{
|
||||
"field": metric.TimeField,
|
||||
intervalField: metric.BucketSize,
|
||||
},
|
||||
dateHistogramAggName: dateHistogramAgg,
|
||||
"aggs": basicAggs,
|
||||
},
|
||||
}
|
||||
|
|
@ -228,20 +276,22 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) {
|
|||
return queryDsl, nil
|
||||
}
|
||||
|
||||
func CollectMetricData(agg interface{}, timeBeforeGroup bool) []insight.MetricData {
|
||||
func CollectMetricData(agg interface{}, timeBeforeGroup bool) ([]insight.MetricData, string) {
|
||||
metricData := []insight.MetricData{}
|
||||
var interval string
|
||||
if timeBeforeGroup {
|
||||
collectMetricDataOther(agg, nil, &metricData, nil)
|
||||
interval = collectMetricDataOther(agg, nil, &metricData, nil)
|
||||
} else {
|
||||
collectMetricData(agg, nil, &metricData)
|
||||
interval = collectMetricData(agg, nil, &metricData)
|
||||
}
|
||||
return metricData
|
||||
return metricData, interval
|
||||
}
|
||||
|
||||
// timeBeforeGroup => false
|
||||
func collectMetricData(agg interface{}, groupValues []string, metricData *[]insight.MetricData) {
|
||||
func collectMetricData(agg interface{}, groupValues []string, metricData *[]insight.MetricData) (interval string){
|
||||
if aggM, ok := agg.(map[string]interface{}); ok {
|
||||
if timeBks, ok := aggM["time_buckets"].(map[string]interface{}); ok {
|
||||
interval, _ = timeBks["interval"].(string)
|
||||
if bks, ok := timeBks["buckets"].([]interface{}); ok {
|
||||
md := insight.MetricData{
|
||||
Data: map[string][]insight.MetricDataItem{},
|
||||
|
|
@ -255,7 +305,7 @@ func collectMetricData(agg interface{}, groupValues []string, metricData *[]insi
|
|||
continue
|
||||
}
|
||||
|
||||
if vm, ok := v.(map[string]interface{}); ok && len(k) < 5 {
|
||||
if vm, ok := v.(map[string]interface{}); ok {
|
||||
collectMetricDataItem(k, vm, &md, bkM["key"])
|
||||
}
|
||||
|
||||
|
|
@ -283,14 +333,12 @@ func collectMetricData(agg interface{}, groupValues []string, metricData *[]insi
|
|||
newGroupValues := make([]string, 0, len(groupValues)+1)
|
||||
newGroupValues = append(newGroupValues, groupValues...)
|
||||
newGroupValues = append(newGroupValues, currentGroup)
|
||||
collectMetricData(bk, newGroupValues, metricData)
|
||||
interval = collectMetricData(bk, newGroupValues, metricData)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
//non time series metric data
|
||||
if len(k) < 5 {
|
||||
collectMetricDataItem(k, vm, &md, nil)
|
||||
}
|
||||
collectMetricDataItem(k, vm, &md, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -299,12 +347,14 @@ func collectMetricData(agg interface{}, groupValues []string, metricData *[]insi
|
|||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// timeBeforeGroup => true
|
||||
func collectMetricDataOther(agg interface{}, groupValues []string, metricData *[]insight.MetricData, timeKey interface{}) {
|
||||
func collectMetricDataOther(agg interface{}, groupValues []string, metricData *[]insight.MetricData, timeKey interface{}) (interval string){
|
||||
if aggM, ok := agg.(map[string]interface{}); ok {
|
||||
if timeBks, ok := aggM["time_buckets"].(map[string]interface{}); ok {
|
||||
interval, _ = timeBks["interval"].(string)
|
||||
if bks, ok := timeBks["buckets"].([]interface{}); ok {
|
||||
md := insight.MetricData{
|
||||
Data: map[string][]insight.MetricDataItem{},
|
||||
|
|
@ -318,7 +368,7 @@ func collectMetricDataOther(agg interface{}, groupValues []string, metricData *[
|
|||
}
|
||||
if vm, ok := v.(map[string]interface{}); ok {
|
||||
if vm["buckets"] != nil {
|
||||
collectMetricDataOther(vm, groupValues, metricData, bkM["key"])
|
||||
interval = collectMetricDataOther(vm, groupValues, metricData, bkM["key"])
|
||||
} else {
|
||||
collectMetricDataItem(k, vm, &md, bkM["key"])
|
||||
}
|
||||
|
|
@ -346,7 +396,7 @@ func collectMetricDataOther(agg interface{}, groupValues []string, metricData *[
|
|||
newGroupValues := make([]string, 0, len(groupValues)+1)
|
||||
newGroupValues = append(newGroupValues, groupValues...)
|
||||
newGroupValues = append(newGroupValues, currentGroup)
|
||||
collectMetricDataOther(bk, newGroupValues, metricData, timeKey)
|
||||
interval = collectMetricDataOther(bk, newGroupValues, metricData, timeKey)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
|
@ -354,7 +404,7 @@ func collectMetricDataOther(agg interface{}, groupValues []string, metricData *[
|
|||
for k, v := range aggM {
|
||||
if vm, ok := v.(map[string]interface{}); ok {
|
||||
if vm["buckets"] != nil {
|
||||
collectMetricDataOther(vm, groupValues, metricData, timeKey)
|
||||
interval = collectMetricDataOther(vm, groupValues, metricData, timeKey)
|
||||
} else {
|
||||
collectMetricDataItem(k, vm, &md, timeKey)
|
||||
}
|
||||
|
|
@ -367,6 +417,7 @@ func collectMetricDataOther(agg interface{}, groupValues []string, metricData *[
|
|||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func collectMetricDataItem(key string, vm map[string]interface{}, metricData *insight.MetricData, timeKey interface{}) {
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ import (
|
|||
"infini.sh/console/core/security/enum"
|
||||
consoleModel "infini.sh/console/model"
|
||||
"infini.sh/console/model/alerting"
|
||||
"infini.sh/console/model/insight"
|
||||
"infini.sh/framework/core/elastic"
|
||||
"infini.sh/framework/core/event"
|
||||
"infini.sh/framework/core/model"
|
||||
|
|
@ -211,6 +212,10 @@ func GetCollectionMetas() map[string]CollectionMeta {
|
|||
},
|
||||
MatchObject: &alerting.Rule{},
|
||||
},
|
||||
"metric": {
|
||||
Name: "metric",
|
||||
MatchObject: &insight.MetricBase{},
|
||||
},
|
||||
}
|
||||
})
|
||||
return collectionMetas
|
||||
|
|
|
|||
Loading…
Reference in New Issue