floraachy
/
console
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update alert rule initialization template

This commit is contained in:
liugq 2023-08-01 17:29:11 +08:00
parent cf8122fe15
commit a1c0d78d44
3 changed files with 139 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
config/initialization.tpl
View File

@ -668,9 +668,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calakp97h710dpnp1fa2
}
],
"format_type": "ratio",
"expression": "avg(payload.elasticsearch.node_stats.process.cpu.percent)",
"title": "CPU Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \nCPU:{{.result_value | to_fixed 2}}%;\n{{end}}"
"expression": "avg(payload.elasticsearch.node_stats.process.cpu.percent)"
},
"conditions": {
"operator": "any",
@ -701,14 +699,17 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calakp97h710dpnp1fa2
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "CPU Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \nCPU:{{.result_value | to_fixed 2}}%;\n{{end}}",
"normal": [
{
"created": "2022-06-16T04:11:10.242061032Z",
"updated": "2022-06-16T04:11:10.242061032Z",
"name": "Slack",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -790,9 +791,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnoaps0
}
],
"format_type": "num",
"expression": "count(payload.elasticsearch.cluster_health.status)",
"title": "Health of Cluster[s] ({{.first_group_value}} ..., {{len .results}} clusters in total) Changed to Red",
"message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}, Name:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }} is RED now;\n{{end}}"
"expression": "count(payload.elasticsearch.cluster_health.status)"
},
"conditions": {
"operator": "any",
@ -807,14 +806,17 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnoaps0
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "Health of Cluster[s] ({{.first_group_value}} ..., {{len .results}} clusters in total) Changed to Red",
"message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}, Name:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }} is RED now;\n{{end}}",
"normal": [
{
"created": "2022-06-16T01:47:11.326727124Z",
"updated": "2022-06-16T01:47:11.326727124Z",
"name": "Slack webhook",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -829,6 +831,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnoaps0
"updated": "2022-06-16T01:47:11.326727124Z",
"name": "DingTalk",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-type": "application/json"
@ -922,9 +925,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnogps1
}
],
"format_type": "ratio",
"expression": "((max(payload.elasticsearch.node_stats.fs.data.total_in_bytes)-max(payload.elasticsearch.node_stats.fs.data.free_in_bytes))/max(payload.elasticsearch.node_stats.fs.data.total_in_bytes))*100",
"title": "Disk Utilization is Too High",
"message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID：{{index .group_values 0}} ;\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID：{{index .group_values 1}} ;\nDisk Usage:{{.result_value | to_fixed 2}}%；Free Storage:{{.relation_values.b | format_bytes 2}}；\n{{end}}"
"expression": "((max(payload.elasticsearch.node_stats.fs.data.total_in_bytes)-max(payload.elasticsearch.node_stats.fs.data.free_in_bytes))/max(payload.elasticsearch.node_stats.fs.data.total_in_bytes))*100"
},
"conditions": {
"operator": "any",
@ -955,14 +956,17 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnogps1
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "Disk Utilization is Too High",
"message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID：{{index .group_values 0}} ;\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID：{{index .group_values 1}} ;\nDisk Usage:{{.result_value | to_fixed 2}}%；Free Storage:{{.relation_values.b | format_bytes 2}}；\n{{end}}",
"normal": [
{
"created": "0001-01-01T00:00:00Z",
"updated": "0001-01-01T00:00:00Z",
"name": "Slack",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -1035,9 +1039,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cbp20n2anisjmu4gehc5
}
],
"format_type": "num",
"expression": "count(metadata.labels.status)",
"title": "Elasticsearch node left cluster",
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \n{{end}}"
"expression": "count(metadata.labels.status)"
},
"conditions": {
"operator": "any",
@ -1052,14 +1054,17 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cbp20n2anisjmu4gehc5
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "Elasticsearch node left cluster",
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \n{{end}}",
"normal": [
{
"created": "2022-08-09T08:52:44.63345561Z",
"updated": "2022-08-09T08:52:44.63345561Z",
"name": "Wechat",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -1133,9 +1138,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calavvp7h710dpnp32r3
}
],
"format_type": "num",
"expression": "count(metadata.index_name)",
"title": "Health of Indices ({{.first_group_value}} ..., {{len .results}} indices in total) Changed to Red",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex name:{{index .group_values 1}}; {{end}}"
"expression": "count(metadata.index_name)"
},
"conditions": {
"operator": "any",
@ -1150,14 +1153,17 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calavvp7h710dpnp32r3
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "Health of Indices ({{.first_group_value}} ..., {{len .results}} indices in total) Changed to Red",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex name:{{index .group_values 1}}; {{end}}",
"normal": [
{
"created": "2022-06-16T04:11:10.242061032Z",
"updated": "2022-06-16T04:11:10.242061032Z",
"name": "Slack",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -1246,9 +1252,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calaqnh7h710dpnp2bm8
}
],
"format_type": "ratio",
"expression": "p90(payload.elasticsearch.node_stats.jvm.mem.heap_used_percent)",
"title": "JVM Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNode name:{{index .group_values 1}}; memory used percent：{{.result_value | to_fixed 2}}%;{{end}}"
"expression": "p90(payload.elasticsearch.node_stats.jvm.mem.heap_used_percent)"
},
"conditions": {
"operator": "any",
@ -1279,14 +1283,17 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calaqnh7h710dpnp2bm8
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "JVM Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNode name:{{index .group_values 1}}; memory used percent：{{.result_value | to_fixed 2}}%;{{end}}",
"normal": [
{
"created": "2022-06-16T04:11:10.242061032Z",
"updated": "2022-06-16T04:11:10.242061032Z",
"name": "Slack",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -1388,9 +1395,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cbp2e4ianisjmu4giqs7
}
],
"format_type": "num",
"expression": "rate(payload.elasticsearch.index_stats.total.search.query_time_in_millis)/rate(payload.elasticsearch.index_stats.primaries.search.query_total)",
"title": "Search latency is great than 500ms",
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex Name:{{index .group_values 1}}; \nCurrent Value:{{.result_value | to_fixed 2}}ms;\n{{end}}"
"expression": "rate(payload.elasticsearch.index_stats.total.search.query_time_in_millis)/rate(payload.elasticsearch.index_stats.primaries.search.query_total)"
},
"conditions": {
"operator": "any",
@ -1405,14 +1410,17 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cbp2e4ianisjmu4giqs7
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "Search latency is great than 500ms",
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex Name:{{index .group_values 1}}; \nCurrent Value:{{.result_value | to_fixed 2}}ms;\n{{end}}",
"normal": [
{
"created": "2022-06-16T04:11:10.242061032Z",
"updated": "2022-06-16T04:11:10.242061032Z",
"name": "Slack",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -1487,9 +1495,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calgapp7h710dpnpbeb6
}
],
"format_type": "bytes",
"expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)",
"title": "Shard Storage >55GB in ({{.first_group_value}} ..., {{len .results}} indices in total)",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }};\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22});\nMax Shard Storage：{{.result_value | format_bytes 2}};\n{{end}}"
"expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)"
},
"conditions": {
"operator": "any",
@ -1504,14 +1510,17 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calgapp7h710dpnpbeb6
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "Shard Storage >55GB in ({{.first_group_value}} ..., {{len .results}} indices in total)",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }};\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22});\nMax Shard Storage：{{.result_value | format_bytes 2}};\n{{end}}",
"normal": [
{
"created": "2022-06-16T04:11:10.242061032Z",
"updated": "2022-06-16T04:11:10.242061032Z",
"name": "Slack",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -1591,9 +1600,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/cb34sfl6psfiqtovhpt4
}
],
"format_type": "ratio",
"expression": "(max(payload.elasticsearch.index_stats.primaries.docs.deleted)/(max(payload.elasticsearch.index_stats.primaries.docs.deleted)+max(payload.elasticsearch.index_stats.primaries.docs.count)))*100",
"title": "Too Many Deleted Documents (>30%)",
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex:{{index .group_values 0}}; \nRatio of Deleted Documents:{{.result_value}};\n{{end}}"
"expression": "(max(payload.elasticsearch.index_stats.primaries.docs.deleted)/(max(payload.elasticsearch.index_stats.primaries.docs.deleted)+max(payload.elasticsearch.index_stats.primaries.docs.count)))*100"
},
"conditions": {
"operator": "any",
@ -1616,14 +1623,17 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/cb34sfl6psfiqtovhpt4
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "Too Many Deleted Documents (>30%)",
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex:{{index .group_values 0}}; \nRatio of Deleted Documents:{{.result_value}};\n{{end}}",
"normal": [
{
"created": "2022-06-16T04:11:10.242061032Z",
"updated": "2022-06-16T04:11:10.242061032Z",
"name": "Slack",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"

82
config/initialization_v5.tpl
View File

@ -612,9 +612,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2
}
],
"format_type": "ratio",
"expression": "avg(payload.elasticsearch.node_stats.process.cpu.percent)",
"title": "CPU Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \nCPU:{{.result_value | to_fixed 2}}%;\n{{end}}"
"expression": "avg(payload.elasticsearch.node_stats.process.cpu.percent)"
},
"conditions": {
"operator": "any",
@ -645,14 +643,17 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "CPU Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \nCPU:{{.result_value | to_fixed 2}}%;\n{{end}}",
"normal": [
{
"created": "2022-06-16T04:11:10.242061032Z",
"updated": "2022-06-16T04:11:10.242061032Z",
"name": "Slack",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -734,9 +735,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0
}
],
"format_type": "num",
"expression": "count(payload.elasticsearch.cluster_health.status)",
"title": "Health of Cluster[s] ({{.first_group_value}} ..., {{len .results}} clusters in total) Changed to Red",
"message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}, Name:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }} is RED now;\n{{end}}"
"expression": "count(payload.elasticsearch.cluster_health.status)"
},
"conditions": {
"operator": "any",
@ -751,14 +750,17 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "Health of Cluster[s] ({{.first_group_value}} ..., {{len .results}} clusters in total) Changed to Red",
"message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}, Name:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }} is RED now;\n{{end}}",
"normal": [
{
"created": "2022-06-16T01:47:11.326727124Z",
"updated": "2022-06-16T01:47:11.326727124Z",
"name": "Slack webhook",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -773,6 +775,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0
"updated": "2022-06-16T01:47:11.326727124Z",
"name": "DingTalk",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-type": "application/json"
@ -866,9 +869,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnogps1
}
],
"format_type": "ratio",
"expression": "((max(payload.elasticsearch.node_stats.fs.data.total_in_bytes)-max(payload.elasticsearch.node_stats.fs.data.free_in_bytes))/max(payload.elasticsearch.node_stats.fs.data.total_in_bytes))*100",
"title": "Disk Utilization is Too High",
"message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID：{{index .group_values 0}} ;\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID：{{index .group_values 1}} ;\nDisk Usage:{{.result_value | to_fixed 2}}%；Free Storage:{{.relation_values.b | format_bytes 2}}；\n{{end}}"
"expression": "((max(payload.elasticsearch.node_stats.fs.data.total_in_bytes)-max(payload.elasticsearch.node_stats.fs.data.free_in_bytes))/max(payload.elasticsearch.node_stats.fs.data.total_in_bytes))*100"
},
"conditions": {
"operator": "any",
@ -899,14 +900,17 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnogps1
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "Disk Utilization is Too High",
"message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID：{{index .group_values 0}} ;\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID：{{index .group_values 1}} ;\nDisk Usage:{{.result_value | to_fixed 2}}%；Free Storage:{{.relation_values.b | format_bytes 2}}；\n{{end}}",
"normal": [
{
"created": "0001-01-01T00:00:00Z",
"updated": "0001-01-01T00:00:00Z",
"name": "Slack",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -979,9 +983,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp20n2anisjmu4gehc5
}
],
"format_type": "num",
"expression": "count(metadata.labels.status)",
"title": "Elasticsearch node left cluster",
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \n{{end}}"
"expression": "count(metadata.labels.status)"
},
"conditions": {
"operator": "any",
@ -996,14 +998,17 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp20n2anisjmu4gehc5
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "Elasticsearch node left cluster",
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \n{{end}}",
"normal": [
{
"created": "2022-08-09T08:52:44.63345561Z",
"updated": "2022-08-09T08:52:44.63345561Z",
"name": "Wechat",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -1077,9 +1082,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calavvp7h710dpnp32r3
}
],
"format_type": "num",
"expression": "count(metadata.index_name)",
"title": "Health of Indices ({{.first_group_value}} ..., {{len .results}} indices in total) Changed to Red",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex name:{{index .group_values 1}}; {{end}}"
"expression": "count(metadata.index_name)"
},
"conditions": {
"operator": "any",
@ -1094,14 +1097,17 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calavvp7h710dpnp32r3
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "Health of Indices ({{.first_group_value}} ..., {{len .results}} indices in total) Changed to Red",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex name:{{index .group_values 1}}; {{end}}",
"normal": [
{
"created": "2022-06-16T04:11:10.242061032Z",
"updated": "2022-06-16T04:11:10.242061032Z",
"name": "Slack",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -1190,9 +1196,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8
}
],
"format_type": "ratio",
"expression": "p90(payload.elasticsearch.node_stats.jvm.mem.heap_used_percent)",
"title": "JVM Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNode name:{{index .group_values 1}}; memory used percent：{{.result_value | to_fixed 2}}%;{{end}}"
"expression": "p90(payload.elasticsearch.node_stats.jvm.mem.heap_used_percent)"
},
"conditions": {
"operator": "any",
@ -1223,14 +1227,17 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "JVM Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNode name:{{index .group_values 1}}; memory used percent：{{.result_value | to_fixed 2}}%;{{end}}",
"normal": [
{
"created": "2022-06-16T04:11:10.242061032Z",
"updated": "2022-06-16T04:11:10.242061032Z",
"name": "Slack",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -1332,9 +1339,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7
}
],
"format_type": "num",
"expression": "rate(payload.elasticsearch.index_stats.total.search.query_time_in_millis)/rate(payload.elasticsearch.index_stats.primaries.search.query_total)",
"title": "Search latency is great than 500ms",
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex Name:{{index .group_values 1}}; \nCurrent Value:{{.result_value | to_fixed 2}}ms;\n{{end}}"
"expression": "rate(payload.elasticsearch.index_stats.total.search.query_time_in_millis)/rate(payload.elasticsearch.index_stats.primaries.search.query_total)"
},
"conditions": {
"operator": "any",
@ -1349,14 +1354,17 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "Search latency is great than 500ms",
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex Name:{{index .group_values 1}}; \nCurrent Value:{{.result_value | to_fixed 2}}ms;\n{{end}}",
"normal": [
{
"created": "2022-06-16T04:11:10.242061032Z",
"updated": "2022-06-16T04:11:10.242061032Z",
"name": "Slack",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -1431,9 +1439,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6
}
],
"format_type": "bytes",
"expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)",
"title": "Shard Storage >55GB in ({{.first_group_value}} ..., {{len .results}} indices in total)",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }};\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22});\nMax Shard Storage：{{.result_value | format_bytes 2}};\n{{end}}"
"expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)"
},
"conditions": {
"operator": "any",
@ -1448,14 +1454,17 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "Shard Storage >55GB in ({{.first_group_value}} ..., {{len .results}} indices in total)",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }};\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22});\nMax Shard Storage：{{.result_value | format_bytes 2}};\n{{end}}",
"normal": [
{
"created": "2022-06-16T04:11:10.242061032Z",
"updated": "2022-06-16T04:11:10.242061032Z",
"name": "Slack",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -1535,9 +1544,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/cb34sfl6psfiqtovhpt4
}
],
"format_type": "ratio",
"expression": "(max(payload.elasticsearch.index_stats.primaries.docs.deleted)/(max(payload.elasticsearch.index_stats.primaries.docs.deleted)+max(payload.elasticsearch.index_stats.primaries.docs.count)))*100",
"title": "Too Many Deleted Documents (>30%)",
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex:{{index .group_values 0}}; \nRatio of Deleted Documents:{{.result_value}};\n{{end}}"
"expression": "(max(payload.elasticsearch.index_stats.primaries.docs.deleted)/(max(payload.elasticsearch.index_stats.primaries.docs.deleted)+max(payload.elasticsearch.index_stats.primaries.docs.count)))*100"
},
"conditions": {
"operator": "any",
@ -1560,14 +1567,17 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/cb34sfl6psfiqtovhpt4
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "Too Many Deleted Documents (>30%)",
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex:{{index .group_values 0}}; \nRatio of Deleted Documents:{{.result_value}};\n{{end}}",
"normal": [
{
"created": "2022-06-16T04:11:10.242061032Z",
"updated": "2022-06-16T04:11:10.242061032Z",
"name": "Slack",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"

84
config/initialization_v6.tpl
View File

@ -621,7 +621,6 @@ PUT $[[INDEX_PREFIX]]activities-00001
}
}
#alerting
POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2
{
@ -688,9 +687,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2
}
],
"format_type": "ratio",
"expression": "avg(payload.elasticsearch.node_stats.process.cpu.percent)",
"title": "CPU Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \nCPU:{{.result_value | to_fixed 2}}%;\n{{end}}"
"expression": "avg(payload.elasticsearch.node_stats.process.cpu.percent)"
},
"conditions": {
"operator": "any",
@ -721,14 +718,17 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "CPU Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \nCPU:{{.result_value | to_fixed 2}}%;\n{{end}}",
"normal": [
{
"created": "2022-06-16T04:11:10.242061032Z",
"updated": "2022-06-16T04:11:10.242061032Z",
"name": "Slack",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -810,9 +810,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0
}
],
"format_type": "num",
"expression": "count(payload.elasticsearch.cluster_health.status)",
"title": "Health of Cluster[s] ({{.first_group_value}} ..., {{len .results}} clusters in total) Changed to Red",
"message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}, Name:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }} is RED now;\n{{end}}"
"expression": "count(payload.elasticsearch.cluster_health.status)"
},
"conditions": {
"operator": "any",
@ -827,14 +825,17 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "Health of Cluster[s] ({{.first_group_value}} ..., {{len .results}} clusters in total) Changed to Red",
"message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}, Name:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }} is RED now;\n{{end}}",
"normal": [
{
"created": "2022-06-16T01:47:11.326727124Z",
"updated": "2022-06-16T01:47:11.326727124Z",
"name": "Slack webhook",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -849,6 +850,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0
"updated": "2022-06-16T01:47:11.326727124Z",
"name": "DingTalk",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-type": "application/json"
@ -942,9 +944,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnogps1
}
],
"format_type": "ratio",
"expression": "((max(payload.elasticsearch.node_stats.fs.data.total_in_bytes)-max(payload.elasticsearch.node_stats.fs.data.free_in_bytes))/max(payload.elasticsearch.node_stats.fs.data.total_in_bytes))*100",
"title": "Disk Utilization is Too High",
"message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID：{{index .group_values 0}} ;\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID：{{index .group_values 1}} ;\nDisk Usage:{{.result_value | to_fixed 2}}%；Free Storage:{{.relation_values.b | format_bytes 2}}；\n{{end}}"
"expression": "((max(payload.elasticsearch.node_stats.fs.data.total_in_bytes)-max(payload.elasticsearch.node_stats.fs.data.free_in_bytes))/max(payload.elasticsearch.node_stats.fs.data.total_in_bytes))*100"
},
"conditions": {
"operator": "any",
@ -975,14 +975,17 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnogps1
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "Disk Utilization is Too High",
"message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID：{{index .group_values 0}} ;\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID：{{index .group_values 1}} ;\nDisk Usage:{{.result_value | to_fixed 2}}%；Free Storage:{{.relation_values.b | format_bytes 2}}；\n{{end}}",
"normal": [
{
"created": "0001-01-01T00:00:00Z",
"updated": "0001-01-01T00:00:00Z",
"name": "Slack",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -1055,9 +1058,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp20n2anisjmu4gehc5
}
],
"format_type": "num",
"expression": "count(metadata.labels.status)",
"title": "Elasticsearch node left cluster",
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \n{{end}}"
"expression": "count(metadata.labels.status)"
},
"conditions": {
"operator": "any",
@ -1072,14 +1073,17 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp20n2anisjmu4gehc5
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "Elasticsearch node left cluster",
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \n{{end}}",
"normal": [
{
"created": "2022-08-09T08:52:44.63345561Z",
"updated": "2022-08-09T08:52:44.63345561Z",
"name": "Wechat",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -1153,9 +1157,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calavvp7h710dpnp32r3
}
],
"format_type": "num",
"expression": "count(metadata.index_name)",
"title": "Health of Indices ({{.first_group_value}} ..., {{len .results}} indices in total) Changed to Red",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex name:{{index .group_values 1}}; {{end}}"
"expression": "count(metadata.index_name)"
},
"conditions": {
"operator": "any",
@ -1170,14 +1172,17 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calavvp7h710dpnp32r3
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "Health of Indices ({{.first_group_value}} ..., {{len .results}} indices in total) Changed to Red",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex name:{{index .group_values 1}}; {{end}}",
"normal": [
{
"created": "2022-06-16T04:11:10.242061032Z",
"updated": "2022-06-16T04:11:10.242061032Z",
"name": "Slack",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -1266,9 +1271,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8
}
],
"format_type": "ratio",
"expression": "p90(payload.elasticsearch.node_stats.jvm.mem.heap_used_percent)",
"title": "JVM Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNode name:{{index .group_values 1}}; memory used percent：{{.result_value | to_fixed 2}}%;{{end}}"
"expression": "p90(payload.elasticsearch.node_stats.jvm.mem.heap_used_percent)"
},
"conditions": {
"operator": "any",
@ -1299,14 +1302,17 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "JVM Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNode name:{{index .group_values 1}}; memory used percent：{{.result_value | to_fixed 2}}%;{{end}}",
"normal": [
{
"created": "2022-06-16T04:11:10.242061032Z",
"updated": "2022-06-16T04:11:10.242061032Z",
"name": "Slack",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -1408,9 +1414,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7
}
],
"format_type": "num",
"expression": "rate(payload.elasticsearch.index_stats.total.search.query_time_in_millis)/rate(payload.elasticsearch.index_stats.primaries.search.query_total)",
"title": "Search latency is great than 500ms",
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex Name:{{index .group_values 1}}; \nCurrent Value:{{.result_value | to_fixed 2}}ms;\n{{end}}"
"expression": "rate(payload.elasticsearch.index_stats.total.search.query_time_in_millis)/rate(payload.elasticsearch.index_stats.primaries.search.query_total)"
},
"conditions": {
"operator": "any",
@ -1425,14 +1429,17 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "Search latency is great than 500ms",
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex Name:{{index .group_values 1}}; \nCurrent Value:{{.result_value | to_fixed 2}}ms;\n{{end}}",
"normal": [
{
"created": "2022-06-16T04:11:10.242061032Z",
"updated": "2022-06-16T04:11:10.242061032Z",
"name": "Slack",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -1507,9 +1514,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6
}
],
"format_type": "bytes",
"expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)",
"title": "Shard Storage >55GB in ({{.first_group_value}} ..., {{len .results}} indices in total)",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }};\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22});\nMax Shard Storage：{{.result_value | format_bytes 2}};\n{{end}}"
"expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)"
},
"conditions": {
"operator": "any",
@ -1524,14 +1529,17 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "Shard Storage >55GB in ({{.first_group_value}} ..., {{len .results}} indices in total)",
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }};\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22});\nMax Shard Storage：{{.result_value | format_bytes 2}};\n{{end}}",
"normal": [
{
"created": "2022-06-16T04:11:10.242061032Z",
"updated": "2022-06-16T04:11:10.242061032Z",
"name": "Slack",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -1611,9 +1619,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/cb34sfl6psfiqtovhpt4
}
],
"format_type": "ratio",
"expression": "(max(payload.elasticsearch.index_stats.primaries.docs.deleted)/(max(payload.elasticsearch.index_stats.primaries.docs.deleted)+max(payload.elasticsearch.index_stats.primaries.docs.count)))*100",
"title": "Too Many Deleted Documents (>30%)",
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex:{{index .group_values 0}}; \nRatio of Deleted Documents:{{.result_value}};\n{{end}}"
"expression": "(max(payload.elasticsearch.index_stats.primaries.docs.deleted)/(max(payload.elasticsearch.index_stats.primaries.docs.deleted)+max(payload.elasticsearch.index_stats.primaries.docs.count)))*100"
},
"conditions": {
"operator": "any",
@ -1636,14 +1642,17 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/cb34sfl6psfiqtovhpt4
}
]
},
"channels": {
"notification_config": {
"enabled": false,
"title": "Too Many Deleted Documents (>30%)",
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex:{{index .group_values 0}}; \nRatio of Deleted Documents:{{.result_value}};\n{{end}}",
"normal": [
{
"created": "2022-06-16T04:11:10.242061032Z",
"updated": "2022-06-16T04:11:10.242061032Z",
"name": "Slack",
"type": "webhook",
"enabled": true,
"webhook": {
"header_params": {
"Content-Type": "application/json"
@ -1664,6 +1673,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/cb34sfl6psfiqtovhpt4
"interval": "1m"
}
}
POST $[[INDEX_PREFIX]]layout/doc/cgjoqud3q95rinbbe1l0
{
"id": "cgjoqud3q95rinbbe1l0",