feat: support function-format parameters in Insight Data API (#93)
* feat: support function-format parameters in Insight Data API * chore: update license header * chore: format code * feat: return `complex_fields` in the view API * feat: include system cluster rollup setting in App Settings API response * chore: update release notes * fix: remove func `sum_func_value_in_group` from using aggregation `date_histogram` * fix: generate unique group id for aggregation name * fix: sort not work when using function `sum_func_value_in_group` * feat: update view template
This commit is contained in:
silenceqi
GitHub
parent
67447b876c
commit
ca243b1ce5
|
|
@ -3567,578 +3567,3 @@ POST $[[SETUP_INDEX_PREFIX]]layout/$[[SETUP_DOC_TYPE]]/cicmhbt3q95ich72lrvg
|
|||
"type": "workspace",
|
||||
"is_fixed": true
|
||||
}
|
||||
|
||||
#shard level
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/bD2jH5QB7KvGccywNCH9
|
||||
{
|
||||
"id": "bD2jH5QB7KvGccywNCH9",
|
||||
"name": "Indexing Rate",
|
||||
"key": "indexing_rate",
|
||||
"level": "shard",
|
||||
"formula": "bD2jH5QB7KvGccywNCH9/{{.bucket_size_in_second}}",
|
||||
"items": [
|
||||
{
|
||||
"name": "bD2jH5QB7KvGccywNCH9",
|
||||
"field": "payload.elasticsearch.shard_stats.indexing.index_total",
|
||||
"statistic": "rate"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "doc/s",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/bD2jH5QB7KvGccywNCH1
|
||||
{
|
||||
"id": "bD2jH5QB7KvGccywNCH1",
|
||||
"name": "Shard Storage",
|
||||
"key": "shard_storage",
|
||||
"level": "shard",
|
||||
"formula": "bD2jH5QB7KvGccywNCH1",
|
||||
"items": [
|
||||
{
|
||||
"name": "bD2jH5QB7KvGccywNCH1",
|
||||
"field": "payload.elasticsearch.shard_stats.store.size_in_bytes",
|
||||
"statistic": "max"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "bytes",
|
||||
"unit": "",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/bD2jH5QB7KvGccywNCH5
|
||||
{
|
||||
"id": "bD2jH5QB7KvGccywNCH5",
|
||||
"name": "Document Count",
|
||||
"key": "doc_count",
|
||||
"level": "shard",
|
||||
"formula": "bD2jH5QB7KvGccywNCH5",
|
||||
"items": [
|
||||
{
|
||||
"name": "bD2jH5QB7KvGccywNCH5",
|
||||
"field": "payload.elasticsearch.shard_stats.docs.count",
|
||||
"statistic": "max"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "number",
|
||||
"unit": "",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/bD2jH5QB7KvGccywNCH2
|
||||
{
|
||||
"id": "bD2jH5QB7KvGccywNCH2",
|
||||
"name": "Search Rate",
|
||||
"key": "search_rate",
|
||||
"level": "shard",
|
||||
"formula": "bD2jH5QB7KvGccywNCH2/{{.bucket_size_in_second}}",
|
||||
"items": [
|
||||
{
|
||||
"name": "bD2jH5QB7KvGccywNCH2",
|
||||
"field": "payload.elasticsearch.shard_stats.search.query_total",
|
||||
"statistic": "rate"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "doc/s",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/bD2jH5QB7KvGccywNCH3
|
||||
{
|
||||
"id": "bD2jH5QB7KvGccywNCH3",
|
||||
"name": "Indexing Latency",
|
||||
"key": "indexing_latency",
|
||||
"level": "shard",
|
||||
"formula": "bD2jH5QB7KvGccywNCx3/bD2jH5QB7KvGccywNCH3",
|
||||
"items": [
|
||||
{
|
||||
"name": "bD2jH5QB7KvGccywNCx3",
|
||||
"field": "payload.elasticsearch.shard_stats.indexing.index_total",
|
||||
"statistic": "rate"
|
||||
},
|
||||
{
|
||||
"name": "bD2jH5QB7KvGccywNCH3",
|
||||
"field": "payload.elasticsearch.shard_stats.indexing.index_time_in_millis",
|
||||
"statistic": "rate"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "ms",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/bD2jH5QB7KvGccywNCH4
|
||||
{
|
||||
"id": "bD2jH5QB7KvGccywNCH4",
|
||||
"name": "Search Latency",
|
||||
"key": "search_latency",
|
||||
"level": "shard",
|
||||
"formula": "bD2jH5QB7KvGccywNCx4/bD2jH5QB7KvGccywNCH4",
|
||||
"items": [
|
||||
{
|
||||
"name": "bD2jH5QB7KvGccywNCH4",
|
||||
"field": "payload.elasticsearch.shard_stats.search.query_total",
|
||||
"statistic": "rate"
|
||||
},
|
||||
{
|
||||
"name": "bD2jH5QB7KvGccywNCx4",
|
||||
"field": "payload.elasticsearch.shard_stats.search.query_time_in_millis",
|
||||
"statistic": "rate"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "ms",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/bD2jH5QB7KvGccywNCH6
|
||||
{
|
||||
"id": "bD2jH5QB7KvGccywNCH6",
|
||||
"name": "Segment Count",
|
||||
"key": "segment_count",
|
||||
"level": "shard",
|
||||
"formula": "bD2jH5QB7KvGccywNCH6",
|
||||
"items": [
|
||||
{
|
||||
"name": "bD2jH5QB7KvGccywNCH6",
|
||||
"field": "payload.elasticsearch.shard_stats.segments.count",
|
||||
"statistic": "max"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "number",
|
||||
"unit": "",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/bD2jH5QB7KvGccywNCH7
|
||||
{
|
||||
"id": "bD2jH5QB7KvGccywNCH7",
|
||||
"name": "Segment memory",
|
||||
"key": "segment_memory",
|
||||
"level": "shard",
|
||||
"formula": "bD2jH5QB7KvGccywNCH7",
|
||||
"items": [
|
||||
{
|
||||
"name": "bD2jH5QB7KvGccywNCH7",
|
||||
"field": "payload.elasticsearch.shard_stats.segments.memory_in_bytes",
|
||||
"statistic": "max"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "number",
|
||||
"unit": "",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
#indices level
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/aD2jH5QB7KvGccywNCH9
|
||||
{
|
||||
"id": "aD2jH5QB7KvGccywNCH9",
|
||||
"name": "Indexing Rate",
|
||||
"key": "indexing_rate",
|
||||
"level": "indices",
|
||||
"formula": "aD2jH5QB7KvGccywNCH9/{{.bucket_size_in_second}}",
|
||||
"items": [
|
||||
{
|
||||
"name": "aD2jH5QB7KvGccywNCH9",
|
||||
"field": "payload.elasticsearch.index_stats.primaries.indexing.index_total",
|
||||
"statistic": "rate"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "doc/s",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/aD2jH5QB7KvGccywNCH1
|
||||
{
|
||||
"id": "aD2jH5QB7KvGccywNCH1",
|
||||
"name": "Index Storage",
|
||||
"key": "index_storage",
|
||||
"level": "indices",
|
||||
"formula": "aD2jH5QB7KvGccywNCH1",
|
||||
"items": [
|
||||
{
|
||||
"name": "aD2jH5QB7KvGccywNCH1",
|
||||
"field": "payload.elasticsearch.index_stats.total.store.size_in_bytes",
|
||||
"statistic": "max"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "bytes",
|
||||
"unit": "",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/aD2jH5QB7KvGccywNCH5
|
||||
{
|
||||
"id": "aD2jH5QB7KvGccywNCH5",
|
||||
"name": "Document Count",
|
||||
"key": "doc_count",
|
||||
"level": "indices",
|
||||
"formula": "aD2jH5QB7KvGccywNCH5",
|
||||
"items": [
|
||||
{
|
||||
"name": "aD2jH5QB7KvGccywNCH5",
|
||||
"field": "payload.elasticsearch.index_stats.total.docs.count",
|
||||
"statistic": "max"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "number",
|
||||
"unit": "",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/aD2jH5QB7KvGccywNCH2
|
||||
{
|
||||
"id": "aD2jH5QB7KvGccywNCH2",
|
||||
"name": "Search Rate",
|
||||
"key": "search_rate",
|
||||
"level": "indices",
|
||||
"formula": "aD2jH5QB7KvGccywNCH2/{{.bucket_size_in_second}}",
|
||||
"items": [
|
||||
{
|
||||
"name": "aD2jH5QB7KvGccywNCH2",
|
||||
"field": "payload.elasticsearch.index_stats.total.search.query_total",
|
||||
"statistic": "rate"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "doc/s",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/aD2jH5QB7KvGccywNCH3
|
||||
{
|
||||
"id": "aD2jH5QB7KvGccywNCH3",
|
||||
"name": "Indexing Latency",
|
||||
"key": "indexing_latency",
|
||||
"level": "indices",
|
||||
"formula": "aD2jH5QB7KvGccywNCx3/aD2jH5QB7KvGccywNCH3",
|
||||
"items": [
|
||||
{
|
||||
"name": "aD2jH5QB7KvGccywNCH3",
|
||||
"field": "payload.elasticsearch.index_stats.primaries.indexing.index_total",
|
||||
"statistic": "rate"
|
||||
},
|
||||
{
|
||||
"name": "aD2jH5QB7KvGccywNCx3",
|
||||
"field": "payload.elasticsearch.index_stats.primaries.indexing.index_time_in_millis",
|
||||
"statistic": "rate"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "ms",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/aD2jH5QB7KvGccywNCH4
|
||||
{
|
||||
"id": "aD2jH5QB7KvGccywNCH4",
|
||||
"name": "Search Latency",
|
||||
"key": "search_latency",
|
||||
"level": "indices",
|
||||
"formula": "aD2jH5QB7KvGccywNCx4/aD2jH5QB7KvGccywNCH4",
|
||||
"items": [
|
||||
{
|
||||
"name": "aD2jH5QB7KvGccywNCH4",
|
||||
"field": "payload.elasticsearch.index_stats.total.search.query_total",
|
||||
"statistic": "rate"
|
||||
},
|
||||
{
|
||||
"name": "aD2jH5QB7KvGccywNCx4",
|
||||
"field": "payload.elasticsearch.index_stats.total.search.query_time_in_millis",
|
||||
"statistic": "rate"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "ms",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/aD2jH5QB7KvGccywNCH6
|
||||
{
|
||||
"id": "aD2jH5QB7KvGccywNCH6",
|
||||
"name": "Segment Count",
|
||||
"key": "segment_count",
|
||||
"level": "indices",
|
||||
"formula": "aD2jH5QB7KvGccywNCH6",
|
||||
"items": [
|
||||
{
|
||||
"name": "aD2jH5QB7KvGccywNCH6",
|
||||
"field": "payload.elasticsearch.index_stats.total.segments.count",
|
||||
"statistic": "max"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "number",
|
||||
"unit": "",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/aD2jH5QB7KvGccywNCH7
|
||||
{
|
||||
"id": "aD2jH5QB7KvGccywNCH7",
|
||||
"name": "Segment memory",
|
||||
"key": "segment_memory",
|
||||
"level": "indices",
|
||||
"formula": "aD2jH5QB7KvGccywNCH7",
|
||||
"items": [
|
||||
{
|
||||
"name": "aD2jH5QB7KvGccywNCH7",
|
||||
"field": "payload.elasticsearch.index_stats.total.segments.memory_in_bytes",
|
||||
"statistic": "max"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "bytes",
|
||||
"unit": "",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
#node level
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/jD2jH5QB7KvGccywNCH9
|
||||
{
|
||||
"id": "jD2jH5QB7KvGccywNCH9",
|
||||
"name": "Indexing Rate",
|
||||
"key": "indexing_rate",
|
||||
"level": "node",
|
||||
"formula": "jD2jH5QB7KvGccywH9/{{.bucket_size_in_second}}",
|
||||
"items": [
|
||||
{
|
||||
"name": "jD2jH5QB7KvGccywH9",
|
||||
"field": "payload.elasticsearch.node_stats.indices.indexing.index_total",
|
||||
"statistic": "derivative"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "doc/s",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/jD2jH5QB7KvGccywNCH4
|
||||
{
|
||||
"id": "jD2jH5QB7KvGccywNCH4",
|
||||
"name": "Process CPU Usage",
|
||||
"key": "process_cpu_used",
|
||||
"level": "node",
|
||||
"formula": "jD2jH5QB7KvGccywNCH4",
|
||||
"items": [
|
||||
{
|
||||
"name": "jD2jH5QB7KvGccywNCH4",
|
||||
"field": "payload.elasticsearch.node_stats.process.cpu.percent",
|
||||
"statistic": "avg"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "",
|
||||
"unit": "%",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/jD2jH5QB7KvGccywNCH3
|
||||
{
|
||||
"id": "jD2jH5QB7KvGccywNCH3",
|
||||
"name": "JVM Heap Usage",
|
||||
"key": "jvm_heap_used",
|
||||
"level": "node",
|
||||
"formula": "jD2jH5QB7KvGccywNCH3",
|
||||
"items": [
|
||||
{
|
||||
"name": "jD2jH5QB7KvGccywNCH3",
|
||||
"field": "payload.elasticsearch.node_stats.jvm.mem.heap_used_in_bytes",
|
||||
"statistic": "max"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "bytes",
|
||||
"unit": "",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/jD2jH5QB7KvGccywNCH1
|
||||
{
|
||||
"id": "jD2jH5QB7KvGccywNCH1",
|
||||
"name": "Indexing Latency",
|
||||
"key": "indexing_latency",
|
||||
"level": "node",
|
||||
"formula": "jD2jH5QB7KvGccywNCx1/jD2jH5QB7KvGccywNCH1",
|
||||
"items": [
|
||||
{
|
||||
"name": "jD2jH5QB7KvGccywNCH1",
|
||||
"field": "payload.elasticsearch.node_stats.indices.indexing.index_total",
|
||||
"statistic": "rate"
|
||||
},
|
||||
{
|
||||
"name": "jD2jH5QB7KvGccywNCx1",
|
||||
"field": "payload.elasticsearch.node_stats.indices.indexing.index_time_in_millis",
|
||||
"statistic": "rate"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "ms",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/jD2jH5QB7KvGccywNCH0
|
||||
{
|
||||
"id": "jD2jH5QB7KvGccywNCH0",
|
||||
"name": "Search Rate",
|
||||
"key": "search_rate",
|
||||
"level": "node",
|
||||
"formula": "jD2jH5QB7KvGccywH0/{{.bucket_size_in_second}}",
|
||||
"items": [
|
||||
{
|
||||
"name": "jD2jH5QB7KvGccywH0",
|
||||
"field": "payload.elasticsearch.node_stats.indices.search.query_total",
|
||||
"statistic": "derivative"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "query/s",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/jD2jH5QB7KvGccywNCH9
|
||||
{
|
||||
"id": "jD2jH5QB7KvGccywNCH9",
|
||||
"name": "Indexing Rate",
|
||||
"key": "indexing_rate",
|
||||
"level": "node",
|
||||
"formula": "jD2jH5QB7KvGccywH9/{{.bucket_size_in_second}}",
|
||||
"items": [
|
||||
{
|
||||
"name": "jD2jH5QB7KvGccywH9",
|
||||
"field": "payload.elasticsearch.node_stats.indices.indexing.index_total",
|
||||
"statistic": "rate"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "doc/s",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/jD2jH5QB7KvGccywNCH5
|
||||
{
|
||||
"id": "jD2jH5QB7KvGccywNCH5",
|
||||
"name": "Search Latency",
|
||||
"key": "search_latency",
|
||||
"level": "node",
|
||||
"formula": "jD2jH5QB7KvGccywNCx5/jD2jH5QB7KvGccywNCH5",
|
||||
"items": [
|
||||
{
|
||||
"name": "jD2jH5QB7KvGccywNCH5",
|
||||
"field": "payload.elasticsearch.node_stats.indices.search.query_total",
|
||||
"statistic": "rate"
|
||||
},
|
||||
{
|
||||
"name": "jD2jH5QB7KvGccywNCx5",
|
||||
"field": "payload.elasticsearch.node_stats.indices.search.query_time_in_millis",
|
||||
"statistic": "rate"
|
||||
}
|
||||
],
|
||||
"statistics": ["rate"],
|
||||
"format": "number",
|
||||
"unit": "ms",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/jD2jH5QB7KvGccywNCH6
|
||||
{
|
||||
"id": "jD2jH5QB7KvGccywNCH6",
|
||||
"name": "Indices Storage",
|
||||
"key": "indices_storage",
|
||||
"level": "node",
|
||||
"formula": "jD2jH5QB7KvGccywNCH6",
|
||||
"items": [
|
||||
{
|
||||
"name": "jD2jH5QB7KvGccywNCH6",
|
||||
"field": "payload.elasticsearch.node_stats.indices.store.size_in_bytes",
|
||||
"statistic": "max"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "bytes",
|
||||
"unit": "",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
PUT $[[SETUP_INDEX_PREFIX]]metric/_doc/jD2jH5QB7KvGccywNCH7
|
||||
{
|
||||
"id": "jD2jH5QB7KvGccywNCH7",
|
||||
"name": "Document Count",
|
||||
"key": "doc_count",
|
||||
"level": "node",
|
||||
"formula": "jD2jH5QB7KvGccywNCH7",
|
||||
"items": [
|
||||
{
|
||||
"name": "jD2jH5QB7KvGccywNCH7",
|
||||
"field": "payload.elasticsearch.node_stats.indices.docs.count",
|
||||
"statistic": "max"
|
||||
}
|
||||
],
|
||||
"statistics": ["max", "min", "sum", "avg", "p99", "medium"],
|
||||
"format": "number",
|
||||
"unit": "",
|
||||
"builtin": true,
|
||||
"created": "2025-01-09T14:30:56.63155+08:00",
|
||||
"updated": "2025-01-09T14:30:56.63155+08:00"
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -11,6 +11,7 @@ Information about release notes of INFINI Console is provided here.
|
|||
|
||||
### Breaking changes
|
||||
### Features
|
||||
- Support function-format parameters in Insight Data API
|
||||
- Support configuring multiple hosts when creating a cluster
|
||||
### Bug fix
|
||||
### Improvements
|
||||
|
|
|
|||
|
|
@ -7,6 +7,14 @@ title: "版本历史"
|
|||
|
||||
这里是 INFINI Console 历史版本发布的相关说明。
|
||||
|
||||
## Latest (In development)
|
||||
|
||||
### Breaking changes
|
||||
### Features
|
||||
- Insight Data API 支持函数格式查询，方便拓展查询功能
|
||||
- 创建集群时支持配置多个主机地址，增强集群的高可用性
|
||||
### Bug fix
|
||||
### Improvements
|
||||
|
||||
## 1.28.0 (2025-01-11)
|
||||
|
||||
|
|
|
|||
24
main.go
24
main.go
|
|
@ -33,6 +33,7 @@ import (
|
|||
"infini.sh/framework/core/api"
|
||||
"infini.sh/framework/core/host"
|
||||
model2 "infini.sh/framework/core/model"
|
||||
"infini.sh/framework/core/util"
|
||||
elastic2 "infini.sh/framework/modules/elastic"
|
||||
_ "time/tzdata"
|
||||
|
||||
|
|
@ -185,6 +186,29 @@ func main() {
|
|||
}
|
||||
return err
|
||||
})
|
||||
api.RegisterAppSetting("system_cluster", func() interface{} {
|
||||
client := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
|
||||
ver := client.GetVersion()
|
||||
if ver.Distribution != elastic.Easysearch {
|
||||
return map[string]interface{}{
|
||||
"rollup_enabled": false,
|
||||
}
|
||||
}
|
||||
settings, err := client.GetClusterSettings(nil)
|
||||
if err != nil {
|
||||
log.Errorf("failed to get cluster settings with system cluster: %v", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
rollupEnabled, _ := util.GetMapValueByKeys([]string{"persistent", "rollup", "search", "enabled"}, settings)
|
||||
rollupEnabledValue := false
|
||||
if v, ok := rollupEnabled.(string); ok && v == "true" {
|
||||
rollupEnabledValue = true
|
||||
}
|
||||
return map[string]interface{}{
|
||||
"rollup_enabled": rollupEnabledValue,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if !global.Env().SetupRequired() {
|
||||
|
|
|
|||
|
|
@ -35,6 +35,29 @@ import (
|
|||
"infini.sh/framework/core/util"
|
||||
)
|
||||
|
||||
const (
|
||||
AggFuncCount = "count"
|
||||
AggFuncAvg = "avg"
|
||||
AggFuncSum = "sum"
|
||||
AggFuncMin = "min"
|
||||
AggFuncMax = "max"
|
||||
AggFuncMedium = "medium"
|
||||
AggFuncValueCount = "value_count"
|
||||
AggFuncCardinality = "cardinality"
|
||||
AggFuncDerivative = "derivative"
|
||||
AggFuncRate = "rate"
|
||||
AggFuncPercent99 = "p99"
|
||||
AggFuncPercent95 = "p95"
|
||||
AggFuncPercent90 = "p90"
|
||||
AggFuncPercent80 = "p80"
|
||||
AggFuncPercent50 = "p50"
|
||||
AggFuncLatest = "latest"
|
||||
AggFuncLatency = "latency"
|
||||
AggFuncSumFuncValueInGroup = "sum_func_value_in_group"
|
||||
AggFuncRateSumFuncValueInGroup = "rate_sum_func_value_in_group"
|
||||
AggFuncLatencySumFuncValueInGroup = "latency_sum_func_value_in_group"
|
||||
)
|
||||
|
||||
type Metric struct {
|
||||
AggTypes []string `json:"agg_types,omitempty"`
|
||||
IndexPattern string `json:"index_pattern,omitempty"`
|
||||
|
|
@ -108,14 +131,55 @@ func (m *Metric) GenerateExpression() (string, error) {
|
|||
|
||||
return string(expressionBytes), nil
|
||||
}
|
||||
func (m *Metric) AutoTimeBeforeGroup() bool {
|
||||
|
||||
// shouldUseAggregation checks if any item's statistic or function exists in the provided aggFuncs list.
|
||||
// If a match is found, it returns true; otherwise, it returns false.
|
||||
func (m *Metric) shouldUseAggregation(aggFuncs []string) bool {
|
||||
for _, item := range m.Items {
|
||||
if item.Statistic == "derivative" {
|
||||
return false
|
||||
// Default to item's Statistic field
|
||||
statistic := item.Statistic
|
||||
|
||||
// If Function is defined, use its first key as the statistic
|
||||
if item.Function != nil {
|
||||
for key := range item.Function {
|
||||
statistic = key
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Check if statistic is in the aggregation function list
|
||||
if util.StringInArray(aggFuncs, statistic) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return true
|
||||
return false
|
||||
}
|
||||
|
||||
// AutoTimeBeforeGroup determines if date aggregation should be applied before terms aggregation.
|
||||
// Returns false if the metric uses any of the specified aggregation functions.
|
||||
func (m *Metric) AutoTimeBeforeGroup() bool {
|
||||
return !m.shouldUseAggregation([]string{
|
||||
AggFuncDerivative,
|
||||
AggFuncRate,
|
||||
AggFuncLatency,
|
||||
AggFuncRateSumFuncValueInGroup,
|
||||
AggFuncLatencySumFuncValueInGroup,
|
||||
})
|
||||
}
|
||||
|
||||
// UseBucketSort determines whether bucket sorting should be used for aggregation.
|
||||
// Returns false if the metric contains specific aggregation functions that require alternative handling.
|
||||
func (m *Metric) UseBucketSort() bool {
|
||||
return m.shouldUseAggregation([]string{
|
||||
AggFuncDerivative,
|
||||
AggFuncRate,
|
||||
AggFuncLatency,
|
||||
AggFuncSumFuncValueInGroup,
|
||||
AggFuncRateSumFuncValueInGroup,
|
||||
AggFuncLatencySumFuncValueInGroup,
|
||||
})
|
||||
}
|
||||
|
||||
func (m *Metric) ValidateSortKey() error {
|
||||
if len(m.Sort) == 0 {
|
||||
return nil
|
||||
|
|
@ -143,6 +207,10 @@ type MetricItem struct {
|
|||
Field string `json:"field"`
|
||||
FieldType string `json:"field_type,omitempty"`
|
||||
Statistic string `json:"statistic,omitempty"`
|
||||
|
||||
//Function specifies the calculation details for the metric,
|
||||
//including the aggregation type and any associated parameters.
|
||||
Function map[string]interface{} `json:"function,omitempty"`
|
||||
}
|
||||
|
||||
type MetricDataItem struct {
|
||||
|
|
|
|||
|
|
@ -128,10 +128,11 @@ func (h *APIHandler) HandleGetViewListAction(w http.ResponseWriter, req *http.Re
|
|||
"title": hit.Source["title"],
|
||||
"viewName": hit.Source["viewName"],
|
||||
},
|
||||
"score": 0,
|
||||
"type": "index-pattern",
|
||||
"namespaces": []string{"default"},
|
||||
"updated_at": hit.Source["updated_at"],
|
||||
"score": 0,
|
||||
"type": "index-pattern",
|
||||
"namespaces": []string{"default"},
|
||||
"updated_at": hit.Source["updated_at"],
|
||||
"complex_fields": hit.Source["complex_fields"],
|
||||
}
|
||||
savedObjects = append(savedObjects, savedObject)
|
||||
}
|
||||
|
|
@ -329,6 +330,8 @@ func (h *APIHandler) HandleBulkGetViewAction(w http.ResponseWriter, req *http.Re
|
|||
"namespaces": []string{"default"},
|
||||
"migrationVersion": map[string]interface{}{"index-pattern": "7.6.0"},
|
||||
"updated_at": hit.Source["updated_at"],
|
||||
"complex_fields": hit.Source["complex_fields"],
|
||||
"references": []interface{}{},
|
||||
}
|
||||
savedObjects = append(savedObjects, savedObject)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,35 @@
|
|||
// Copyright (C) INFINI Labs & INFINI LIMITED.
|
||||
//
|
||||
// The INFINI Console is offered under the GNU Affero General Public License v3.0
|
||||
// and as commercial software.
|
||||
//
|
||||
// For commercial licensing, contact us at:
|
||||
// - Website: infinilabs.com
|
||||
// - Email: hello@infini.ltd
|
||||
//
|
||||
// Open Source licensed under AGPL V3:
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
/* Copyright © INFINI Ltd. All rights reserved.
|
||||
* Web: https://infinilabs.com
|
||||
* Email: hello#infini.ltd */
|
||||
|
||||
package insight
|
||||
|
||||
type Function interface {
|
||||
//GenerateAggregation generates aggregation for specify function, e.g. rate, latency ...
|
||||
//
|
||||
// metricName: The name of the metric to be calculated (used as an identifier in the aggregation).
|
||||
GenerateAggregation(metricName string) (map[string]interface{}, error)
|
||||
}
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
// Copyright (C) INFINI Labs & INFINI LIMITED.
|
||||
//
|
||||
// The INFINI Console is offered under the GNU Affero General Public License v3.0
|
||||
// and as commercial software.
|
||||
//
|
||||
// For commercial licensing, contact us at:
|
||||
// - Website: infinilabs.com
|
||||
// - Email: hello@infini.ltd
|
||||
//
|
||||
// Open Source licensed under AGPL V3:
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
/* Copyright © INFINI Ltd. All rights reserved.
|
||||
* Web: https://infinilabs.com
|
||||
* Email: hello#infini.ltd */
|
||||
|
||||
package function
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"infini.sh/framework/core/util"
|
||||
)
|
||||
|
||||
// Latency represents a function that calculates latency as a ratio of two metrics.
|
||||
// Divisor: The denominator in the latency calculation.
|
||||
// Dividend: The numerator in the latency calculation.
|
||||
type Latency struct {
|
||||
Divisor string `json:"divisor"`
|
||||
Dividend string `json:"dividend"`
|
||||
}
|
||||
|
||||
// GenerateAggregation implements the `Function` interface and generates an aggregation for the `latency` function.
|
||||
func (p *Latency) GenerateAggregation(metricName string) (map[string]interface{}, error) {
|
||||
if p.Dividend == "" || p.Divisor == "" {
|
||||
return nil, fmt.Errorf("empty divisor or dividend for agg func: latency")
|
||||
}
|
||||
|
||||
var (
|
||||
divisorAggID = util.GetUUID()
|
||||
dividendAggID = util.GetUUID()
|
||||
divisorBaseAggID = util.GetUUID()
|
||||
dividendBaseAggID = util.GetUUID()
|
||||
)
|
||||
return util.MapStr{
|
||||
dividendBaseAggID: util.MapStr{
|
||||
"max": util.MapStr{
|
||||
"field": p.Dividend,
|
||||
},
|
||||
},
|
||||
dividendAggID: util.MapStr{
|
||||
"derivative": util.MapStr{
|
||||
"buckets_path": dividendBaseAggID,
|
||||
},
|
||||
},
|
||||
divisorBaseAggID: util.MapStr{
|
||||
"max": util.MapStr{
|
||||
"field": p.Divisor,
|
||||
},
|
||||
},
|
||||
divisorAggID: util.MapStr{
|
||||
"derivative": util.MapStr{
|
||||
"buckets_path": divisorBaseAggID,
|
||||
},
|
||||
},
|
||||
metricName: util.MapStr{
|
||||
"bucket_script": util.MapStr{
|
||||
"buckets_path": util.MapStr{
|
||||
"dividend": dividendAggID,
|
||||
"divisor": divisorAggID,
|
||||
},
|
||||
"script": "params.dividend / params.divisor",
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
// Copyright (C) INFINI Labs & INFINI LIMITED.
|
||||
//
|
||||
// The INFINI Console is offered under the GNU Affero General Public License v3.0
|
||||
// and as commercial software.
|
||||
//
|
||||
// For commercial licensing, contact us at:
|
||||
// - Website: infinilabs.com
|
||||
// - Email: hello@infini.ltd
|
||||
//
|
||||
// Open Source licensed under AGPL V3:
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
/* Copyright © INFINI Ltd. All rights reserved.
|
||||
* Web: https://infinilabs.com
|
||||
* Email: hello#infini.ltd */
|
||||
|
||||
package function
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"infini.sh/framework/core/util"
|
||||
)
|
||||
|
||||
// LatencySumFuncValueInGroup represents a function that calculates the latency as a sum of values in a grouped aggregation.
|
||||
// It combines a `Latency` calculation with a grouping operation and applies a specified aggregation function within each group.
|
||||
// The group defines the field to group by, the aggregation function to apply, and the maximum number of groups to include.
|
||||
type LatencySumFuncValueInGroup struct {
|
||||
Latency // Embeds the Latency struct for fields `Divisor` and `Dividend`.
|
||||
Group struct {
|
||||
Field string `json:"field"` // The field used for grouping.
|
||||
Func string `json:"func"` // The aggregation function to apply within each group (e.g., max, min, sum).
|
||||
Size uint `json:"size"` // The maximum number of groups to return.
|
||||
} `json:"group"`
|
||||
}
|
||||
|
||||
// GenerateAggregation implements the `Function` interface and generates an aggregation for the `latency_sum_func_value_in_group` function.
|
||||
func (p *LatencySumFuncValueInGroup) GenerateAggregation(metricName string) (map[string]interface{}, error) {
|
||||
// Validate that the divisor and dividend are not empty.
|
||||
if p.Dividend == "" || p.Divisor == "" {
|
||||
return nil, fmt.Errorf("empty divisor or dividend for agg func: latency")
|
||||
}
|
||||
|
||||
// Validate that the group field is not empty.
|
||||
if p.Group.Field == "" {
|
||||
return nil, fmt.Errorf("empty group field for agg func: latency_sum_func_value_in_group")
|
||||
}
|
||||
|
||||
// Set default function to `max` if no function is provided.
|
||||
if p.Group.Func == "" {
|
||||
p.Group.Func = "max"
|
||||
}
|
||||
|
||||
// Set default size for the group if none is provided.
|
||||
if p.Group.Size == 0 {
|
||||
p.Group.Size = 65536 // Default to a large size to include most groups.
|
||||
}
|
||||
var (
|
||||
divisorBaseAggID = util.GetUUID()
|
||||
dividendBaseAggID = util.GetUUID()
|
||||
sumDivisorAggID = util.GetUUID()
|
||||
sumDividendAggID = util.GetUUID()
|
||||
derivativeDivisorAggID = util.GetUUID()
|
||||
derivativeDividendAggID = util.GetUUID()
|
||||
groupID = fmt.Sprintf("sum_group_%s", util.GetUUID())
|
||||
)
|
||||
return util.MapStr{
|
||||
groupID: util.MapStr{
|
||||
"aggs": util.MapStr{
|
||||
divisorBaseAggID: util.MapStr{
|
||||
p.Group.Func: util.MapStr{
|
||||
"field": p.Divisor,
|
||||
},
|
||||
},
|
||||
dividendBaseAggID: util.MapStr{
|
||||
p.Group.Func: util.MapStr{
|
||||
"field": p.Dividend,
|
||||
},
|
||||
},
|
||||
},
|
||||
"terms": util.MapStr{
|
||||
"field": p.Group.Field,
|
||||
"size": p.Group.Size,
|
||||
},
|
||||
},
|
||||
sumDivisorAggID: util.MapStr{
|
||||
"sum_bucket": util.MapStr{
|
||||
"buckets_path": fmt.Sprintf("%s>%s", groupID, divisorBaseAggID),
|
||||
},
|
||||
},
|
||||
sumDividendAggID: util.MapStr{
|
||||
"sum_bucket": util.MapStr{
|
||||
"buckets_path": fmt.Sprintf("%s>%s", groupID, dividendBaseAggID),
|
||||
},
|
||||
},
|
||||
derivativeDivisorAggID: util.MapStr{
|
||||
"derivative": util.MapStr{
|
||||
"buckets_path": sumDivisorAggID,
|
||||
},
|
||||
},
|
||||
derivativeDividendAggID: util.MapStr{
|
||||
"derivative": util.MapStr{
|
||||
"buckets_path": sumDividendAggID,
|
||||
},
|
||||
},
|
||||
metricName: util.MapStr{
|
||||
"bucket_script": util.MapStr{
|
||||
"buckets_path": util.MapStr{
|
||||
"dividend": derivativeDividendAggID,
|
||||
"divisor": derivativeDivisorAggID,
|
||||
},
|
||||
"script": "params.dividend / params.divisor",
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
// Copyright (C) INFINI Labs & INFINI LIMITED.
|
||||
//
|
||||
// The INFINI Console is offered under the GNU Affero General Public License v3.0
|
||||
// and as commercial software.
|
||||
//
|
||||
// For commercial licensing, contact us at:
|
||||
// - Website: infinilabs.com
|
||||
// - Email: hello@infini.ltd
|
||||
//
|
||||
// Open Source licensed under AGPL V3:
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
/* Copyright © INFINI Ltd. All rights reserved.
|
||||
* Web: https://infinilabs.com
|
||||
* Email: hello#infini.ltd */
|
||||
|
||||
package function
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"infini.sh/framework/core/util"
|
||||
)
|
||||
|
||||
// Base represents a common structure for functions that operate on a specific field.
|
||||
// Field: The name of the field to be aggregated.
|
||||
type Base struct {
|
||||
Field string `json:"field"` // The field on which the aggregation operates.
|
||||
}
|
||||
|
||||
// Rate extends the Base structure to include scaling for rate calculations.
|
||||
// Scale: A multiplier applied to the rate calculation for normalization.
|
||||
type Rate struct {
|
||||
Base
|
||||
Scale uint `json:"scale"` // The scaling factor used in the rate calculation.
|
||||
}
|
||||
|
||||
// GenerateAggregation implements the `Function` interface and generates an aggregation for the `rate` function.
|
||||
func (p *Rate) GenerateAggregation(metricName string) (map[string]interface{}, error) {
|
||||
if p.Field == "" {
|
||||
return nil, fmt.Errorf("empty field for agg func: rate")
|
||||
}
|
||||
if p.Scale == 0 {
|
||||
p.Scale = 1
|
||||
}
|
||||
var (
|
||||
aggID = util.GetUUID()
|
||||
derivativeAggID = util.GetUUID()
|
||||
)
|
||||
return util.MapStr{
|
||||
aggID: util.MapStr{
|
||||
"max": util.MapStr{
|
||||
"field": p.Field,
|
||||
},
|
||||
},
|
||||
derivativeAggID: util.MapStr{
|
||||
"derivative": util.MapStr{
|
||||
"buckets_path": aggID,
|
||||
},
|
||||
},
|
||||
metricName: util.MapStr{
|
||||
"bucket_script": util.MapStr{
|
||||
"buckets_path": util.MapStr{
|
||||
"value": derivativeAggID,
|
||||
},
|
||||
"script": util.MapStr{
|
||||
"source": "params.value * params.scale",
|
||||
"params": util.MapStr{
|
||||
"scale": p.Scale,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
// Copyright (C) INFINI Labs & INFINI LIMITED.
|
||||
//
|
||||
// The INFINI Console is offered under the GNU Affero General Public License v3.0
|
||||
// and as commercial software.
|
||||
//
|
||||
// For commercial licensing, contact us at:
|
||||
// - Website: infinilabs.com
|
||||
// - Email: hello@infini.ltd
|
||||
//
|
||||
// Open Source licensed under AGPL V3:
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
/* Copyright © INFINI Ltd. All rights reserved.
|
||||
* Web: https://infinilabs.com
|
||||
* Email: hello#infini.ltd */
|
||||
|
||||
package function
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"infini.sh/framework/core/util"
|
||||
)
|
||||
|
||||
// RateSumFuncValueInGroup represents a function that calculates the rate as a sum of values in a grouped aggregation.
|
||||
// It combines a `Rate` calculation with a grouping operation and applies a specified aggregation function within each group.
|
||||
// The group defines the field to group by, the aggregation function to apply, and the maximum number of groups to include.
|
||||
type RateSumFuncValueInGroup struct {
|
||||
Rate // Embeds the Rate struct for fields `Field` and `Scale`.
|
||||
Group struct {
|
||||
Field string `json:"field"` // The field used for grouping.
|
||||
Func string `json:"func"` // The aggregation function to apply within each group (e.g., max, min, sum).
|
||||
Size uint `json:"size"` // The maximum number of groups to return.
|
||||
} `json:"group"`
|
||||
}
|
||||
|
||||
// GenerateAggregation implements the `Function` interface and generates an aggregation for the `rate_sum_func_value_in_group` function.
|
||||
// This function constructs an aggregation pipeline that computes the sum of values for a given field within groups, calculates the derivative,
|
||||
// and then applies a bucket script to scale the resulting value by a provided `scale` factor.
|
||||
func (p *RateSumFuncValueInGroup) GenerateAggregation(metricName string) (map[string]interface{}, error) {
|
||||
// Validate that the field is provided for the aggregation.
|
||||
if p.Field == "" {
|
||||
return nil, fmt.Errorf("empty field for agg func: rate_sum_func_value_in_group")
|
||||
}
|
||||
|
||||
// Set default scale factor to 1 if not specified.
|
||||
if p.Scale == 0 {
|
||||
p.Scale = 1
|
||||
}
|
||||
|
||||
// Validate that the group field is provided.
|
||||
if p.Group.Field == "" {
|
||||
return nil, fmt.Errorf("empty group field for agg func: rate_sum_func_value_in_group")
|
||||
}
|
||||
|
||||
// Set default function to `max` if no function is specified for grouping.
|
||||
if p.Group.Func == "" {
|
||||
p.Group.Func = "max"
|
||||
}
|
||||
|
||||
// Set default size for the group if none is specified.
|
||||
if p.Group.Size == 0 {
|
||||
p.Group.Size = 65536 // Default to a large size to include most groups.
|
||||
}
|
||||
var (
|
||||
aggID = util.GetUUID()
|
||||
sumBucketAggID = util.GetUUID()
|
||||
derivativeAggID = util.GetUUID()
|
||||
groupID = fmt.Sprintf("sum_group_%s", util.GetUUID())
|
||||
)
|
||||
return util.MapStr{
|
||||
groupID: util.MapStr{
|
||||
"aggs": util.MapStr{
|
||||
aggID: util.MapStr{
|
||||
p.Group.Func: util.MapStr{
|
||||
"field": p.Field,
|
||||
},
|
||||
},
|
||||
},
|
||||
"terms": util.MapStr{
|
||||
"field": p.Group.Field,
|
||||
"size": p.Group.Size,
|
||||
},
|
||||
},
|
||||
sumBucketAggID: util.MapStr{
|
||||
"sum_bucket": util.MapStr{
|
||||
"buckets_path": fmt.Sprintf("%s>%s", groupID, aggID),
|
||||
},
|
||||
},
|
||||
derivativeAggID: util.MapStr{
|
||||
"derivative": util.MapStr{
|
||||
"buckets_path": sumBucketAggID,
|
||||
},
|
||||
},
|
||||
metricName: util.MapStr{
|
||||
"bucket_script": util.MapStr{
|
||||
"buckets_path": util.MapStr{
|
||||
"value": derivativeAggID,
|
||||
},
|
||||
"script": util.MapStr{
|
||||
"source": "params.value * params.scale",
|
||||
"params": util.MapStr{
|
||||
"scale": p.Scale,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
// Copyright (C) INFINI Labs & INFINI LIMITED.
|
||||
//
|
||||
// The INFINI Console is offered under the GNU Affero General Public License v3.0
|
||||
// and as commercial software.
|
||||
//
|
||||
// For commercial licensing, contact us at:
|
||||
// - Website: infinilabs.com
|
||||
// - Email: hello@infini.ltd
|
||||
//
|
||||
// Open Source licensed under AGPL V3:
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
/* Copyright © INFINI Ltd. All rights reserved.
|
||||
* Web: https://infinilabs.com
|
||||
* Email: hello#infini.ltd */
|
||||
|
||||
package function
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"infini.sh/framework/core/util"
|
||||
)
|
||||
|
||||
// SumFuncValueInGroup represents a function to calculate a sum of values in a grouped aggregation.
|
||||
// It combines a `Rate` calculation with a grouping operation and a specified aggregation function within each group.
|
||||
// The group defines the field to group by, the aggregation function to apply, and the maximum number of groups to include.
|
||||
type SumFuncValueInGroup struct {
|
||||
Rate // Embeds the Rate struct for rate-specific fields like `Field` and `Scale`.
|
||||
Group struct {
|
||||
Field string `json:"field"` // The field used for grouping.
|
||||
Func string `json:"func"` // The aggregation function to apply within each group (e.g., max, min, sum).
|
||||
Size uint `json:"size"` // The maximum number of groups to return.
|
||||
} `json:"group"`
|
||||
}
|
||||
|
||||
// GenerateAggregation implements the `Function` interface and generates an aggregation for the `sum_func_value_in_group` function.
|
||||
func (p *SumFuncValueInGroup) GenerateAggregation(metricName string) (map[string]interface{}, error) {
|
||||
if p.Group.Field == "" {
|
||||
return nil, fmt.Errorf("empty group field for agg func: sum_func_value_in_group")
|
||||
}
|
||||
// Set a default aggregation function if none is specified.
|
||||
if p.Group.Func == "" {
|
||||
p.Group.Func = "max"
|
||||
}
|
||||
// Set a default size for the group if none is specified.
|
||||
if p.Group.Size == 0 {
|
||||
p.Group.Size = 65536 // Default to a large size to include most groups.
|
||||
}
|
||||
aggID := util.GetUUID()
|
||||
groupID := fmt.Sprintf("sum_group_%s", util.GetUUID())
|
||||
return util.MapStr{
|
||||
groupID: util.MapStr{
|
||||
"aggs": util.MapStr{
|
||||
aggID: util.MapStr{
|
||||
p.Group.Func: util.MapStr{
|
||||
"field": p.Field,
|
||||
},
|
||||
},
|
||||
},
|
||||
"terms": util.MapStr{
|
||||
"field": p.Group.Field,
|
||||
"size": p.Group.Size,
|
||||
},
|
||||
},
|
||||
metricName: util.MapStr{
|
||||
"sum_bucket": util.MapStr{
|
||||
"buckets_path": fmt.Sprintf("%s>%s", groupID, aggID),
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
|
@ -29,6 +29,7 @@ package insight
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"infini.sh/console/plugin/api/insight/function"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
|
|
@ -37,31 +38,36 @@ import (
|
|||
"infini.sh/framework/core/util"
|
||||
)
|
||||
|
||||
func generateAgg(metricItem *insight.MetricItem, timeField string) map[string]interface{} {
|
||||
func generateAgg(metricItem *insight.MetricItem, timeField string) (map[string]interface{}, error) {
|
||||
var (
|
||||
aggType = "value_count"
|
||||
field = metricItem.Field
|
||||
)
|
||||
if field == "" || field == "*" {
|
||||
field = "_id"
|
||||
if timeField != "" {
|
||||
field = timeField
|
||||
} else {
|
||||
field = "_id"
|
||||
}
|
||||
}
|
||||
var percent = 0.0
|
||||
var isPipeline = false
|
||||
|
||||
switch metricItem.Statistic {
|
||||
case "max", "min", "sum", "avg", "cardinality":
|
||||
case insight.AggFuncMax, insight.AggFuncMin, insight.AggFuncSum, insight.AggFuncAvg, insight.AggFuncCardinality:
|
||||
aggType = metricItem.Statistic
|
||||
case "count", "value_count":
|
||||
case insight.AggFuncCount, insight.AggFuncValueCount:
|
||||
aggType = "value_count"
|
||||
case "derivative":
|
||||
case insight.AggFuncDerivative:
|
||||
aggType = "max"
|
||||
isPipeline = true
|
||||
case "medium": // from es version 6.6
|
||||
case insight.AggFuncMedium: // from es version 6.6
|
||||
aggType = "median_absolute_deviation"
|
||||
case "p99", "p95", "p90", "p80", "p50":
|
||||
case insight.AggFuncPercent99, insight.AggFuncPercent95, insight.AggFuncPercent90, insight.AggFuncPercent80, insight.AggFuncPercent50:
|
||||
aggType = "percentiles"
|
||||
percentStr := strings.TrimPrefix(metricItem.Statistic, "p")
|
||||
percent, _ = strconv.ParseFloat(percentStr, 32)
|
||||
case "latest":
|
||||
case insight.AggFuncLatest:
|
||||
aggType = "top_hits"
|
||||
}
|
||||
aggValue := util.MapStr{}
|
||||
|
|
@ -89,7 +95,7 @@ func generateAgg(metricItem *insight.MetricItem, timeField string) map[string]in
|
|||
},
|
||||
}
|
||||
if !isPipeline {
|
||||
return aggs
|
||||
return aggs, nil
|
||||
}
|
||||
pipelineAggID := util.GetUUID()
|
||||
aggs[pipelineAggID] = aggs[metricItem.Name]
|
||||
|
|
@ -98,7 +104,53 @@ func generateAgg(metricItem *insight.MetricItem, timeField string) map[string]in
|
|||
"buckets_path": pipelineAggID,
|
||||
},
|
||||
}
|
||||
return aggs
|
||||
return aggs, nil
|
||||
}
|
||||
|
||||
func generateAggByFunction(metricItem *insight.MetricItem, timeField string) (map[string]interface{}, error) {
|
||||
if metricItem.Function == nil {
|
||||
return nil, fmt.Errorf("empty function for metric item: %s", metricItem.Name)
|
||||
}
|
||||
if len(metricItem.Function) != 1 {
|
||||
return nil, fmt.Errorf("invalid function for metric item: %s: expected exactly one function name, but got zero or multiple", metricItem.Name)
|
||||
}
|
||||
var (
|
||||
funcName string
|
||||
funcParams interface{}
|
||||
)
|
||||
for k, v := range metricItem.Function {
|
||||
funcName = k
|
||||
funcParams = v
|
||||
}
|
||||
if funcParams == nil {
|
||||
return nil, fmt.Errorf("empty params for agg func: %s", funcName)
|
||||
}
|
||||
funcName = strings.ToLower(funcName)
|
||||
buf := util.MustToJSONBytes(funcParams)
|
||||
var generator Function
|
||||
switch funcName {
|
||||
case insight.AggFuncRate:
|
||||
generator = &function.Rate{}
|
||||
case insight.AggFuncLatency:
|
||||
generator = &function.Latency{}
|
||||
case insight.AggFuncSumFuncValueInGroup:
|
||||
generator = &function.SumFuncValueInGroup{}
|
||||
case insight.AggFuncRateSumFuncValueInGroup:
|
||||
generator = &function.RateSumFuncValueInGroup{}
|
||||
case insight.AggFuncLatencySumFuncValueInGroup:
|
||||
generator = &function.LatencySumFuncValueInGroup{}
|
||||
default:
|
||||
baseParams := function.Base{}
|
||||
util.MustFromJSONBytes(buf, &baseParams)
|
||||
newMetricItem := &insight.MetricItem{
|
||||
Name: metricItem.Name,
|
||||
Field: baseParams.Field,
|
||||
Statistic: funcName,
|
||||
}
|
||||
return generateAgg(newMetricItem, timeField)
|
||||
}
|
||||
util.MustFromJSONBytes(buf, &generator)
|
||||
return generator.GenerateAggregation(metricItem.Name)
|
||||
}
|
||||
|
||||
func GenerateQuery(metric *insight.Metric) (interface{}, error) {
|
||||
|
|
@ -109,7 +161,18 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) {
|
|||
if metricItem.Name == "" {
|
||||
metricItem.Name = string(rune('a' + i))
|
||||
}
|
||||
metricAggs := generateAgg(&metricItem, metric.TimeField)
|
||||
var (
|
||||
metricAggs map[string]interface{}
|
||||
err error
|
||||
)
|
||||
if metricItem.Function != nil {
|
||||
metricAggs, err = generateAggByFunction(&metricItem, metric.TimeField)
|
||||
} else {
|
||||
metricAggs, err = generateAgg(&metricItem, metric.TimeField)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := util.MergeFields(basicAggs, metricAggs, true); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -181,19 +244,24 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) {
|
|||
}
|
||||
if i == grpLength-1 && len(metric.Sort) > 0 {
|
||||
//use bucket sort instead of terms order when time after group
|
||||
if !timeBeforeGroup && len(metric.Sort) > 0 {
|
||||
basicAggs["sort_field"] = util.MapStr{
|
||||
"max_bucket": util.MapStr{
|
||||
"buckets_path": fmt.Sprintf("time_buckets>%s", metric.Sort[0].Key),
|
||||
},
|
||||
if metric.UseBucketSort() && len(metric.Sort) > 0 {
|
||||
sortKey := metric.Sort[0].Key
|
||||
if !timeBeforeGroup {
|
||||
sortKey = "sort_field"
|
||||
basicAggs[sortKey] = util.MapStr{
|
||||
"max_bucket": util.MapStr{
|
||||
"buckets_path": fmt.Sprintf("time_buckets>%s", metric.Sort[0].Key),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
//using 65536 as a workaround for the terms aggregation limit; the actual limit is enforced in the bucket sort step
|
||||
termsCfg["size"] = 65536
|
||||
basicAggs["bucket_sorter"] = util.MapStr{
|
||||
"bucket_sort": util.MapStr{
|
||||
"size": limit,
|
||||
"sort": []util.MapStr{
|
||||
{"sort_field": util.MapStr{"order": metric.Sort[0].Direction}},
|
||||
{sortKey: util.MapStr{"order": metric.Sort[0].Direction}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
|
@ -214,7 +282,9 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) {
|
|||
}
|
||||
termsOrder = append(termsOrder, util.MapStr{sortKey: sortItem.Direction})
|
||||
}
|
||||
termsCfg["order"] = termsOrder
|
||||
if len(termsOrder) > 0 {
|
||||
termsCfg["order"] = termsOrder
|
||||
}
|
||||
}
|
||||
}
|
||||
groupAgg := util.MapStr{
|
||||
|
|
|
|||
Loading…
Reference in New Issue