update alerting initialization template
This commit is contained in:
liugq
parent
d76b2295b4
commit
4c949e350a
|
|
@ -669,8 +669,8 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calakp97h710dpnp1fa2
|
|||
],
|
||||
"format_type": "ratio",
|
||||
"expression": "avg(payload.elasticsearch.node_stats.process.cpu.percent)",
|
||||
"title": "CPU Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%",
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};NodeID:{{index .group_values 1}}; CPU:{{.result_value | to_fixed 2}}%;\n{{end}}"
|
||||
"title": "CPU Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%",
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \nCPU:{{.result_value | to_fixed 2}}%;\n{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -702,7 +702,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calakp97h710dpnp1fa2
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-06-16T04:11:10.242061032Z",
|
||||
|
|
@ -714,8 +714,8 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calakp97h710dpnp1fa2
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -792,7 +792,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnoaps0
|
|||
"format_type": "num",
|
||||
"expression": "count(payload.elasticsearch.cluster_health.status)",
|
||||
"title": "Health of Cluster[s] ({{.first_group_value}} ..., {{len .results}} clusters in total) Changed to Red",
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}} is red now;\n{{end}}"
|
||||
"message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}, Name:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }} is RED now;\n{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -808,7 +808,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnoaps0
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-06-16T01:47:11.326727124Z",
|
||||
|
|
@ -820,8 +820,8 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnoaps0
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"high\"}} \"#EB4C21\" {{else if eq .priority \"medium\"}} \"#FFB449\" {{else if eq .priority \"low\"}} \"#87d068\" {{else}} \"#2db7f5\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|View Cluster Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|View Cluster Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -834,8 +834,8 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnoaps0
|
|||
"Content-type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${DINGTALK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\"msgtype\": \"text\",\"text\": {\"content\":\"Alerting: \\n{{.title}}\\n\\n{{.message}}\\nLink:${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}\"}}"
|
||||
"url": "{{.env.DINGTALK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\"msgtype\": \"text\",\"text\": {\"content\":\"Alerting: \\n{{.title}}\\n\\n{{.message}}\\nLink:{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"}}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -924,7 +924,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnogps1
|
|||
"format_type": "ratio",
|
||||
"expression": "((max(payload.elasticsearch.node_stats.fs.data.total_in_bytes)-max(payload.elasticsearch.node_stats.fs.data.free_in_bytes))/max(payload.elasticsearch.node_stats.fs.data.total_in_bytes))*100",
|
||||
"title": "Disk Utilization is Too High",
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID：{{index .group_values 0}} ;\nNodeID：{{index .group_values 1}} ;\nDisk Usage:{{.result_value | to_fixed 2}}%；Free Storage:{{.relation_values.b | format_bytes 2}}；\n{{end}}"
|
||||
"message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID：{{index .group_values 0}} ;\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID：{{index .group_values 1}} ;\nDisk Usage:{{.result_value | to_fixed 2}}%；Free Storage:{{.relation_values.b | format_bytes 2}}；\n{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -956,7 +956,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnogps1
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "0001-01-01T00:00:00Z",
|
||||
|
|
@ -968,8 +968,8 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnogps1
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Free:* {{.relation_values.b | format_bytes 2}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Free:* {{.relation_values.b | format_bytes 2}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -1037,7 +1037,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cbp20n2anisjmu4gehc5
|
|||
"format_type": "num",
|
||||
"expression": "count(metadata.labels.status)",
|
||||
"title": "Elasticsearch node left cluster",
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nNodeID:{{index .group_values 1}}; \n{{end}}"
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \n{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -1053,7 +1053,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cbp20n2anisjmu4gehc5
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-08-09T08:52:44.63345561Z",
|
||||
|
|
@ -1065,8 +1065,8 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cbp20n2anisjmu4gehc5
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${WECHAT_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"Incident [#{{.event_id}}](${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}) is ongoing\\n{{.title}}\\n\n {{range .results}}\n >ClusterID:<font color=\\\"comment\\\">{{index .group_values 0}}</font>\n >NodeID:<font color=\\\"comment\\\">{{index .group_values 1}}</font>\n >Priority:<font color=\\\"comment\\\">{{.priority}}</font>\n >Link:[View Cluster Monitoring](${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}) \n {{end}}\"\n }\n}\n"
|
||||
"url": "{{.env.WECHAT_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"Incident [#{{.event_id}}]({{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n{{.title}}\\n\n {{range .results}}\n >ClusterID:<font color=\\\"comment\\\">{{index .group_values 0}}</font>\n >NodeID:<font color=\\\"comment\\\">{{index .group_values 1}}</font>\n >Priority:<font color=\\\"comment\\\">{{.priority}}</font>\n >Link:[View Cluster Monitoring]({{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}) \n {{end}}\"\n }\n}\n"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -1135,7 +1135,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calavvp7h710dpnp32r3
|
|||
"format_type": "num",
|
||||
"expression": "count(metadata.index_name)",
|
||||
"title": "Health of Indices ({{.first_group_value}} ..., {{len .results}} indices in total) Changed to Red",
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; Index name:{{index .group_values 1}}; {{end}}"
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex name:{{index .group_values 1}}; {{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -1151,7 +1151,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calavvp7h710dpnp32r3
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-06-16T04:11:10.242061032Z",
|
||||
|
|
@ -1163,8 +1163,8 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calavvp7h710dpnp32r3
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}?_g=%7B%22tab%22%3A%22indices%22%7D|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}?_g=%7B%22tab%22%3A%22indices%22%7D|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -1248,7 +1248,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calaqnh7h710dpnp2bm8
|
|||
"format_type": "ratio",
|
||||
"expression": "p90(payload.elasticsearch.node_stats.jvm.mem.heap_used_percent)",
|
||||
"title": "JVM Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%",
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; Node name:{{index .group_values 1}}; memory used percent：{{.result_value | to_fixed 2}}%;{{end}}"
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNode name:{{index .group_values 1}}; memory used percent：{{.result_value | to_fixed 2}}%;{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -1280,7 +1280,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calaqnh7h710dpnp2bm8
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-06-16T04:11:10.242061032Z",
|
||||
|
|
@ -1292,8 +1292,8 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calaqnh7h710dpnp2bm8
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -1390,7 +1390,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cbp2e4ianisjmu4giqs7
|
|||
"format_type": "num",
|
||||
"expression": "rate(payload.elasticsearch.index_stats.total.search.query_time_in_millis)/rate(payload.elasticsearch.index_stats.primaries.search.query_total)",
|
||||
"title": "Search latency is great than 500ms",
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nIndex name:{{index .group_values 1}}; \nCurrent value:{{.result_value | to_fixed 2}}ms;\n{{end}}"
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex Name:{{index .group_values 1}}; \nCurrent Value:{{.result_value | to_fixed 2}}ms;\n{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -1406,7 +1406,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cbp2e4ianisjmu4giqs7
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-06-16T04:11:10.242061032Z",
|
||||
|
|
@ -1418,8 +1418,8 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cbp2e4ianisjmu4giqs7
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Latency:* {{.result_value | to_fixed 2}}ms\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Latency:* {{.result_value | to_fixed 2}}ms\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -1489,7 +1489,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calgapp7h710dpnpbeb6
|
|||
"format_type": "bytes",
|
||||
"expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)",
|
||||
"title": "Shard Storage >55GB in ({{.first_group_value}} ..., {{len .results}} indices in total)",
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; Index:{{index .group_values 1}}; Max Shard Storage：{{.result_value | format_bytes 2}};{{end}}"
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }};\nIndex: [{{index .group_values 1}}]({{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22});\nMax Shard Storage：{{.result_value | format_bytes 2}};\n{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -1505,7 +1505,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calgapp7h710dpnpbeb6
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-06-16T04:11:10.242061032Z",
|
||||
|
|
@ -1517,8 +1517,8 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calgapp7h710dpnpbeb6
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Max Shard Storage:* {{.result_value | format_bytes 2}}\"\n },\n \n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Max Shard Storage:* {{.result_value | format_bytes 2}}\"\n },\n \n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -1593,7 +1593,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/cb34sfl6psfiqtovhpt4
|
|||
"format_type": "ratio",
|
||||
"expression": "(max(payload.elasticsearch.index_stats.primaries.docs.deleted)/(max(payload.elasticsearch.index_stats.primaries.docs.deleted)+max(payload.elasticsearch.index_stats.primaries.docs.count)))*100",
|
||||
"title": "Too Many Deleted Documents (>30%)",
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nIndex:{{index .group_values 0}}; \nRatio of Deleted Documents:{{.result_value}};\n{{end}}"
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex:{{index .group_values 0}}; \nRatio of Deleted Documents:{{.result_value}};\n{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -1617,7 +1617,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/cb34sfl6psfiqtovhpt4
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-06-16T04:11:10.242061032Z",
|
||||
|
|
@ -1629,8 +1629,8 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/cb34sfl6psfiqtovhpt4
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Deleted:* {{.result_value | to_fixed 2}}%\"\n },\n \n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Deleted:* {{.result_value | to_fixed 2}}%\"\n },\n \n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
|
|||
|
|
@ -111,10 +111,6 @@ PUT _template/$[[INDEX_PREFIX]]logs-rollover
|
|||
"settings": {
|
||||
"index": {
|
||||
"format": "7",
|
||||
"lifecycle": {
|
||||
"name" : "ilm_$[[INDEX_PREFIX]]metrics-30days-retention",
|
||||
"rollover_alias" : "$[[INDEX_PREFIX]]logs"
|
||||
},
|
||||
"codec": "best_compression",
|
||||
"number_of_shards": "1",
|
||||
"translog": {
|
||||
|
|
@ -164,9 +160,7 @@ PUT $[[INDEX_PREFIX]]logs-00001
|
|||
PUT _template/$[[INDEX_PREFIX]]requests_logging-rollover
|
||||
{
|
||||
"order": 100000,
|
||||
"index_patterns": [
|
||||
"$[[INDEX_PREFIX]]requests_logging*"
|
||||
],
|
||||
"template": "$[[INDEX_PREFIX]]requests_logging*",
|
||||
"settings": {
|
||||
"index": {
|
||||
"format": "7",
|
||||
|
|
@ -298,10 +292,6 @@ PUT _template/$[[INDEX_PREFIX]]alert-history-rollover
|
|||
"settings" : {
|
||||
"index" : {
|
||||
"format" : "7",
|
||||
"lifecycle" : {
|
||||
"name" : "ilm_$[[INDEX_PREFIX]]metrics-30days-retention",
|
||||
"rollover_alias" : "$[[INDEX_PREFIX]]alert-history"
|
||||
},
|
||||
"codec" : "best_compression",
|
||||
"number_of_shards" : "1",
|
||||
"translog.durability":"async"
|
||||
|
|
@ -422,12 +412,7 @@ PUT $[[INDEX_PREFIX]]alert-history-00001
|
|||
},
|
||||
"search_text" : {
|
||||
"type" : "text",
|
||||
"analyzer" : "suggest_text_search",
|
||||
"index_prefixes" : {
|
||||
"min_chars" : 2,
|
||||
"max_chars" : 5
|
||||
},
|
||||
"index_phrases" : true
|
||||
"analyzer" : "suggest_text_search"
|
||||
},
|
||||
"severity" : {
|
||||
"type" : "keyword"
|
||||
|
|
@ -451,14 +436,10 @@ PUT $[[INDEX_PREFIX]]alert-history-00001
|
|||
PUT _template/$[[INDEX_PREFIX]]activities-rollover
|
||||
{
|
||||
"order" : 100000,
|
||||
"index_patterns" : "$[[INDEX_PREFIX]]activities*",
|
||||
"template" : "$[[INDEX_PREFIX]]activities*",
|
||||
"settings" : {
|
||||
"index" : {
|
||||
"format" : "7",
|
||||
"lifecycle" : {
|
||||
"name" : "ilm_$[[INDEX_PREFIX]]metrics-30days-retention",
|
||||
"rollover_alias" : "$[[INDEX_PREFIX]]activities"
|
||||
},
|
||||
"codec" : "best_compression",
|
||||
"number_of_shards" : "1",
|
||||
"translog.durability":"async"
|
||||
|
|
@ -571,12 +552,12 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2
|
|||
"id": "builtin-calakp97h710dpnp1fa2",
|
||||
"created": "2022-06-16T03:58:29.437447113Z",
|
||||
"updated": "2022-07-21T23:12:51.111569117Z",
|
||||
"name": "CPU utilization is Too High",
|
||||
"enabled": false,
|
||||
"creator": {
|
||||
"name": "$[[USERNAME]]",
|
||||
"id": "$[[USER_ID]]"
|
||||
},
|
||||
},
|
||||
"name": "CPU utilization is Too High",
|
||||
"enabled": false,
|
||||
"resource": {
|
||||
"resource_id": "$[[RESOURCE_ID]]",
|
||||
"resource_name": "$[[RESOURCE_NAME]]",
|
||||
|
|
@ -632,8 +613,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2
|
|||
],
|
||||
"format_type": "ratio",
|
||||
"expression": "avg(payload.elasticsearch.node_stats.process.cpu.percent)",
|
||||
"title": "CPU Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%",
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};NodeID:{{index .group_values 1}}; CPU:{{.result_value | to_fixed 2}}%;\n{{end}}"
|
||||
"title": "CPU Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%",
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \nCPU:{{.result_value | to_fixed 2}}%;\n{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -665,7 +646,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-06-16T04:11:10.242061032Z",
|
||||
|
|
@ -677,8 +658,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -704,7 +685,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0
|
|||
"creator": {
|
||||
"name": "$[[USERNAME]]",
|
||||
"id": "$[[USER_ID]]"
|
||||
},
|
||||
},
|
||||
"resource": {
|
||||
"resource_id": "$[[RESOURCE_ID]]",
|
||||
"resource_name": "$[[RESOURCE_NAME]]",
|
||||
|
|
@ -755,7 +736,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0
|
|||
"format_type": "num",
|
||||
"expression": "count(payload.elasticsearch.cluster_health.status)",
|
||||
"title": "Health of Cluster[s] ({{.first_group_value}} ..., {{len .results}} clusters in total) Changed to Red",
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}} is red now;\n{{end}}"
|
||||
"message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}, Name:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }} is RED now;\n{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -771,7 +752,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-06-16T01:47:11.326727124Z",
|
||||
|
|
@ -783,8 +764,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"high\"}} \"#EB4C21\" {{else if eq .priority \"medium\"}} \"#FFB449\" {{else if eq .priority \"low\"}} \"#87d068\" {{else}} \"#2db7f5\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|View Cluster Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|View Cluster Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -797,8 +778,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0
|
|||
"Content-type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${DINGTALK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\"msgtype\": \"text\",\"text\": {\"content\":\"Alerting: \\n{{.title}}\\n\\n{{.message}}\\nLink:${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}\"}}"
|
||||
"url": "{{.env.DINGTALK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\"msgtype\": \"text\",\"text\": {\"content\":\"Alerting: \\n{{.title}}\\n\\n{{.message}}\\nLink:{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"}}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -825,7 +806,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnogps1
|
|||
"creator": {
|
||||
"name": "$[[USERNAME]]",
|
||||
"id": "$[[USER_ID]]"
|
||||
},
|
||||
},
|
||||
"resource": {
|
||||
"resource_id": "$[[RESOURCE_ID]]",
|
||||
"resource_name": "$[[RESOURCE_NAME]]",
|
||||
|
|
@ -887,7 +868,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnogps1
|
|||
"format_type": "ratio",
|
||||
"expression": "((max(payload.elasticsearch.node_stats.fs.data.total_in_bytes)-max(payload.elasticsearch.node_stats.fs.data.free_in_bytes))/max(payload.elasticsearch.node_stats.fs.data.total_in_bytes))*100",
|
||||
"title": "Disk Utilization is Too High",
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID：{{index .group_values 0}} ;\nNodeID：{{index .group_values 1}} ;\nDisk Usage:{{.result_value | to_fixed 2}}%；Free Storage:{{.relation_values.b | format_bytes 2}}；\n{{end}}"
|
||||
"message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID：{{index .group_values 0}} ;\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID：{{index .group_values 1}} ;\nDisk Usage:{{.result_value | to_fixed 2}}%；Free Storage:{{.relation_values.b | format_bytes 2}}；\n{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -919,7 +900,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnogps1
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "0001-01-01T00:00:00Z",
|
||||
|
|
@ -931,8 +912,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnogps1
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Free:* {{.relation_values.b | format_bytes 2}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Free:* {{.relation_values.b | format_bytes 2}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -958,7 +939,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp20n2anisjmu4gehc5
|
|||
"creator": {
|
||||
"name": "$[[USERNAME]]",
|
||||
"id": "$[[USER_ID]]"
|
||||
},
|
||||
},
|
||||
"resource": {
|
||||
"resource_id": "$[[RESOURCE_ID]]",
|
||||
"resource_name": "$[[RESOURCE_NAME]]",
|
||||
|
|
@ -1000,7 +981,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp20n2anisjmu4gehc5
|
|||
"format_type": "num",
|
||||
"expression": "count(metadata.labels.status)",
|
||||
"title": "Elasticsearch node left cluster",
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nNodeID:{{index .group_values 1}}; \n{{end}}"
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \n{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -1016,7 +997,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp20n2anisjmu4gehc5
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-08-09T08:52:44.63345561Z",
|
||||
|
|
@ -1028,8 +1009,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp20n2anisjmu4gehc5
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${WECHAT_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"Incident [#{{.event_id}}](${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}) is ongoing\\n{{.title}}\\n\n {{range .results}}\n >ClusterID:<font color=\\\"comment\\\">{{index .group_values 0}}</font>\n >NodeID:<font color=\\\"comment\\\">{{index .group_values 1}}</font>\n >Priority:<font color=\\\"comment\\\">{{.priority}}</font>\n >Link:[View Cluster Monitoring](${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}) \n {{end}}\"\n }\n}\n"
|
||||
"url": "{{.env.WECHAT_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"Incident [#{{.event_id}}]({{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n{{.title}}\\n\n {{range .results}}\n >ClusterID:<font color=\\\"comment\\\">{{index .group_values 0}}</font>\n >NodeID:<font color=\\\"comment\\\">{{index .group_values 1}}</font>\n >Priority:<font color=\\\"comment\\\">{{.priority}}</font>\n >Link:[View Cluster Monitoring]({{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}) \n {{end}}\"\n }\n}\n"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -1056,7 +1037,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calavvp7h710dpnp32r3
|
|||
"creator": {
|
||||
"name": "$[[USERNAME]]",
|
||||
"id": "$[[USER_ID]]"
|
||||
},
|
||||
},
|
||||
"resource": {
|
||||
"resource_id": "$[[RESOURCE_ID]]",
|
||||
"resource_name": "$[[RESOURCE_NAME]]",
|
||||
|
|
@ -1098,7 +1079,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calavvp7h710dpnp32r3
|
|||
"format_type": "num",
|
||||
"expression": "count(metadata.index_name)",
|
||||
"title": "Health of Indices ({{.first_group_value}} ..., {{len .results}} indices in total) Changed to Red",
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; Index name:{{index .group_values 1}}; {{end}}"
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex name:{{index .group_values 1}}; {{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -1114,7 +1095,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calavvp7h710dpnp32r3
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-06-16T04:11:10.242061032Z",
|
||||
|
|
@ -1126,8 +1107,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calavvp7h710dpnp32r3
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}?_g=%7B%22tab%22%3A%22indices%22%7D|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}?_g=%7B%22tab%22%3A%22indices%22%7D|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -1154,7 +1135,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8
|
|||
"creator": {
|
||||
"name": "$[[USERNAME]]",
|
||||
"id": "$[[USER_ID]]"
|
||||
},
|
||||
},
|
||||
"resource": {
|
||||
"resource_id": "$[[RESOURCE_ID]]",
|
||||
"resource_name": "$[[RESOURCE_NAME]]",
|
||||
|
|
@ -1211,7 +1192,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8
|
|||
"format_type": "ratio",
|
||||
"expression": "p90(payload.elasticsearch.node_stats.jvm.mem.heap_used_percent)",
|
||||
"title": "JVM Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%",
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; Node name:{{index .group_values 1}}; memory used percent：{{.result_value | to_fixed 2}}%;{{end}}"
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNode name:{{index .group_values 1}}; memory used percent：{{.result_value | to_fixed 2}}%;{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -1243,7 +1224,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-06-16T04:11:10.242061032Z",
|
||||
|
|
@ -1255,8 +1236,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -1282,7 +1263,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7
|
|||
"creator": {
|
||||
"name": "$[[USERNAME]]",
|
||||
"id": "$[[USER_ID]]"
|
||||
},
|
||||
},
|
||||
"resource": {
|
||||
"resource_id": "$[[RESOURCE_ID]]",
|
||||
"resource_name": "$[[RESOURCE_NAME]]",
|
||||
|
|
@ -1353,7 +1334,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7
|
|||
"format_type": "num",
|
||||
"expression": "rate(payload.elasticsearch.index_stats.total.search.query_time_in_millis)/rate(payload.elasticsearch.index_stats.primaries.search.query_total)",
|
||||
"title": "Search latency is great than 500ms",
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nIndex name:{{index .group_values 1}}; \nCurrent value:{{.result_value | to_fixed 2}}ms;\n{{end}}"
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex Name:{{index .group_values 1}}; \nCurrent Value:{{.result_value | to_fixed 2}}ms;\n{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -1369,7 +1350,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-06-16T04:11:10.242061032Z",
|
||||
|
|
@ -1381,8 +1362,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Latency:* {{.result_value | to_fixed 2}}ms\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Latency:* {{.result_value | to_fixed 2}}ms\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -1408,7 +1389,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6
|
|||
"creator": {
|
||||
"name": "$[[USERNAME]]",
|
||||
"id": "$[[USER_ID]]"
|
||||
},
|
||||
},
|
||||
"resource": {
|
||||
"resource_id": "$[[RESOURCE_ID]]",
|
||||
"resource_name": "$[[RESOURCE_NAME]]",
|
||||
|
|
@ -1452,7 +1433,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6
|
|||
"format_type": "bytes",
|
||||
"expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)",
|
||||
"title": "Shard Storage >55GB in ({{.first_group_value}} ..., {{len .results}} indices in total)",
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; Index:{{index .group_values 1}}; Max Shard Storage：{{.result_value | format_bytes 2}};{{end}}"
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }};\nIndex: [{{index .group_values 1}}]({{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22});\nMax Shard Storage：{{.result_value | format_bytes 2}};\n{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -1468,7 +1449,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-06-16T04:11:10.242061032Z",
|
||||
|
|
@ -1480,8 +1461,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Max Shard Storage:* {{.result_value | format_bytes 2}}\"\n },\n \n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Max Shard Storage:* {{.result_value | format_bytes 2}}\"\n },\n \n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -1507,7 +1488,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/cb34sfl6psfiqtovhpt4
|
|||
"creator": {
|
||||
"name": "$[[USERNAME]]",
|
||||
"id": "$[[USER_ID]]"
|
||||
},
|
||||
},
|
||||
"resource": {
|
||||
"resource_id": "$[[RESOURCE_ID]]",
|
||||
"resource_name": "$[[RESOURCE_NAME]]",
|
||||
|
|
@ -1556,7 +1537,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/cb34sfl6psfiqtovhpt4
|
|||
"format_type": "ratio",
|
||||
"expression": "(max(payload.elasticsearch.index_stats.primaries.docs.deleted)/(max(payload.elasticsearch.index_stats.primaries.docs.deleted)+max(payload.elasticsearch.index_stats.primaries.docs.count)))*100",
|
||||
"title": "Too Many Deleted Documents (>30%)",
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nIndex:{{index .group_values 0}}; \nRatio of Deleted Documents:{{.result_value}};\n{{end}}"
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex:{{index .group_values 0}}; \nRatio of Deleted Documents:{{.result_value}};\n{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -1580,7 +1561,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/cb34sfl6psfiqtovhpt4
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-06-16T04:11:10.242061032Z",
|
||||
|
|
@ -1592,8 +1573,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/cb34sfl6psfiqtovhpt4
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Deleted:* {{.result_value | to_fixed 2}}%\"\n },\n \n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Deleted:* {{.result_value | to_fixed 2}}%\"\n },\n \n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
|
|||
|
|
@ -628,12 +628,12 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2
|
|||
"id": "builtin-calakp97h710dpnp1fa2",
|
||||
"created": "2022-06-16T03:58:29.437447113Z",
|
||||
"updated": "2022-07-21T23:12:51.111569117Z",
|
||||
"name": "CPU utilization is Too High",
|
||||
"enabled": false,
|
||||
"creator": {
|
||||
"name": "$[[USERNAME]]",
|
||||
"id": "$[[USER_ID]]"
|
||||
},
|
||||
},
|
||||
"name": "CPU utilization is Too High",
|
||||
"enabled": false,
|
||||
"resource": {
|
||||
"resource_id": "$[[RESOURCE_ID]]",
|
||||
"resource_name": "$[[RESOURCE_NAME]]",
|
||||
|
|
@ -689,8 +689,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2
|
|||
],
|
||||
"format_type": "ratio",
|
||||
"expression": "avg(payload.elasticsearch.node_stats.process.cpu.percent)",
|
||||
"title": "CPU Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%",
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};NodeID:{{index .group_values 1}}; CPU:{{.result_value | to_fixed 2}}%;\n{{end}}"
|
||||
"title": "CPU Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%",
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \nCPU:{{.result_value | to_fixed 2}}%;\n{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -722,7 +722,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-06-16T04:11:10.242061032Z",
|
||||
|
|
@ -734,8 +734,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -761,7 +761,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0
|
|||
"creator": {
|
||||
"name": "$[[USERNAME]]",
|
||||
"id": "$[[USER_ID]]"
|
||||
},
|
||||
},
|
||||
"resource": {
|
||||
"resource_id": "$[[RESOURCE_ID]]",
|
||||
"resource_name": "$[[RESOURCE_NAME]]",
|
||||
|
|
@ -812,7 +812,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0
|
|||
"format_type": "num",
|
||||
"expression": "count(payload.elasticsearch.cluster_health.status)",
|
||||
"title": "Health of Cluster[s] ({{.first_group_value}} ..., {{len .results}} clusters in total) Changed to Red",
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}} is red now;\n{{end}}"
|
||||
"message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}, Name:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }} is RED now;\n{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -828,7 +828,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-06-16T01:47:11.326727124Z",
|
||||
|
|
@ -840,8 +840,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"high\"}} \"#EB4C21\" {{else if eq .priority \"medium\"}} \"#FFB449\" {{else if eq .priority \"low\"}} \"#87d068\" {{else}} \"#2db7f5\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|View Cluster Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|View Cluster Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -854,8 +854,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0
|
|||
"Content-type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${DINGTALK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\"msgtype\": \"text\",\"text\": {\"content\":\"Alerting: \\n{{.title}}\\n\\n{{.message}}\\nLink:${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}\"}}"
|
||||
"url": "{{.env.DINGTALK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\"msgtype\": \"text\",\"text\": {\"content\":\"Alerting: \\n{{.title}}\\n\\n{{.message}}\\nLink:{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"}}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -882,7 +882,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnogps1
|
|||
"creator": {
|
||||
"name": "$[[USERNAME]]",
|
||||
"id": "$[[USER_ID]]"
|
||||
},
|
||||
},
|
||||
"resource": {
|
||||
"resource_id": "$[[RESOURCE_ID]]",
|
||||
"resource_name": "$[[RESOURCE_NAME]]",
|
||||
|
|
@ -944,7 +944,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnogps1
|
|||
"format_type": "ratio",
|
||||
"expression": "((max(payload.elasticsearch.node_stats.fs.data.total_in_bytes)-max(payload.elasticsearch.node_stats.fs.data.free_in_bytes))/max(payload.elasticsearch.node_stats.fs.data.total_in_bytes))*100",
|
||||
"title": "Disk Utilization is Too High",
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID：{{index .group_values 0}} ;\nNodeID：{{index .group_values 1}} ;\nDisk Usage:{{.result_value | to_fixed 2}}%；Free Storage:{{.relation_values.b | format_bytes 2}}；\n{{end}}"
|
||||
"message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID：{{index .group_values 0}} ;\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID：{{index .group_values 1}} ;\nDisk Usage:{{.result_value | to_fixed 2}}%；Free Storage:{{.relation_values.b | format_bytes 2}}；\n{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -976,7 +976,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnogps1
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "0001-01-01T00:00:00Z",
|
||||
|
|
@ -988,8 +988,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnogps1
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Free:* {{.relation_values.b | format_bytes 2}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Free:* {{.relation_values.b | format_bytes 2}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -1015,7 +1015,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp20n2anisjmu4gehc5
|
|||
"creator": {
|
||||
"name": "$[[USERNAME]]",
|
||||
"id": "$[[USER_ID]]"
|
||||
},
|
||||
},
|
||||
"resource": {
|
||||
"resource_id": "$[[RESOURCE_ID]]",
|
||||
"resource_name": "$[[RESOURCE_NAME]]",
|
||||
|
|
@ -1057,7 +1057,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp20n2anisjmu4gehc5
|
|||
"format_type": "num",
|
||||
"expression": "count(metadata.labels.status)",
|
||||
"title": "Elasticsearch node left cluster",
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nNodeID:{{index .group_values 1}}; \n{{end}}"
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \n{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -1073,7 +1073,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp20n2anisjmu4gehc5
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-08-09T08:52:44.63345561Z",
|
||||
|
|
@ -1085,8 +1085,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp20n2anisjmu4gehc5
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${WECHAT_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"Incident [#{{.event_id}}](${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}) is ongoing\\n{{.title}}\\n\n {{range .results}}\n >ClusterID:<font color=\\\"comment\\\">{{index .group_values 0}}</font>\n >NodeID:<font color=\\\"comment\\\">{{index .group_values 1}}</font>\n >Priority:<font color=\\\"comment\\\">{{.priority}}</font>\n >Link:[View Cluster Monitoring](${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}) \n {{end}}\"\n }\n}\n"
|
||||
"url": "{{.env.WECHAT_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"Incident [#{{.event_id}}]({{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n{{.title}}\\n\n {{range .results}}\n >ClusterID:<font color=\\\"comment\\\">{{index .group_values 0}}</font>\n >NodeID:<font color=\\\"comment\\\">{{index .group_values 1}}</font>\n >Priority:<font color=\\\"comment\\\">{{.priority}}</font>\n >Link:[View Cluster Monitoring]({{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}) \n {{end}}\"\n }\n}\n"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -1113,7 +1113,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calavvp7h710dpnp32r3
|
|||
"creator": {
|
||||
"name": "$[[USERNAME]]",
|
||||
"id": "$[[USER_ID]]"
|
||||
},
|
||||
},
|
||||
"resource": {
|
||||
"resource_id": "$[[RESOURCE_ID]]",
|
||||
"resource_name": "$[[RESOURCE_NAME]]",
|
||||
|
|
@ -1155,7 +1155,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calavvp7h710dpnp32r3
|
|||
"format_type": "num",
|
||||
"expression": "count(metadata.index_name)",
|
||||
"title": "Health of Indices ({{.first_group_value}} ..., {{len .results}} indices in total) Changed to Red",
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; Index name:{{index .group_values 1}}; {{end}}"
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex name:{{index .group_values 1}}; {{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -1171,7 +1171,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calavvp7h710dpnp32r3
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-06-16T04:11:10.242061032Z",
|
||||
|
|
@ -1183,8 +1183,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calavvp7h710dpnp32r3
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}?_g=%7B%22tab%22%3A%22indices%22%7D|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}?_g=%7B%22tab%22%3A%22indices%22%7D|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -1211,7 +1211,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8
|
|||
"creator": {
|
||||
"name": "$[[USERNAME]]",
|
||||
"id": "$[[USER_ID]]"
|
||||
},
|
||||
},
|
||||
"resource": {
|
||||
"resource_id": "$[[RESOURCE_ID]]",
|
||||
"resource_name": "$[[RESOURCE_NAME]]",
|
||||
|
|
@ -1268,7 +1268,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8
|
|||
"format_type": "ratio",
|
||||
"expression": "p90(payload.elasticsearch.node_stats.jvm.mem.heap_used_percent)",
|
||||
"title": "JVM Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%",
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; Node name:{{index .group_values 1}}; memory used percent：{{.result_value | to_fixed 2}}%;{{end}}"
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNode name:{{index .group_values 1}}; memory used percent：{{.result_value | to_fixed 2}}%;{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -1300,7 +1300,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-06-16T04:11:10.242061032Z",
|
||||
|
|
@ -1312,8 +1312,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -1339,7 +1339,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7
|
|||
"creator": {
|
||||
"name": "$[[USERNAME]]",
|
||||
"id": "$[[USER_ID]]"
|
||||
},
|
||||
},
|
||||
"resource": {
|
||||
"resource_id": "$[[RESOURCE_ID]]",
|
||||
"resource_name": "$[[RESOURCE_NAME]]",
|
||||
|
|
@ -1410,7 +1410,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7
|
|||
"format_type": "num",
|
||||
"expression": "rate(payload.elasticsearch.index_stats.total.search.query_time_in_millis)/rate(payload.elasticsearch.index_stats.primaries.search.query_total)",
|
||||
"title": "Search latency is great than 500ms",
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nIndex name:{{index .group_values 1}}; \nCurrent value:{{.result_value | to_fixed 2}}ms;\n{{end}}"
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex Name:{{index .group_values 1}}; \nCurrent Value:{{.result_value | to_fixed 2}}ms;\n{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -1426,7 +1426,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-06-16T04:11:10.242061032Z",
|
||||
|
|
@ -1438,8 +1438,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Latency:* {{.result_value | to_fixed 2}}ms\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Latency:* {{.result_value | to_fixed 2}}ms\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -1465,7 +1465,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6
|
|||
"creator": {
|
||||
"name": "$[[USERNAME]]",
|
||||
"id": "$[[USER_ID]]"
|
||||
},
|
||||
},
|
||||
"resource": {
|
||||
"resource_id": "$[[RESOURCE_ID]]",
|
||||
"resource_name": "$[[RESOURCE_NAME]]",
|
||||
|
|
@ -1509,7 +1509,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6
|
|||
"format_type": "bytes",
|
||||
"expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)",
|
||||
"title": "Shard Storage >55GB in ({{.first_group_value}} ..., {{len .results}} indices in total)",
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; Index:{{index .group_values 1}}; Max Shard Storage：{{.result_value | format_bytes 2}};{{end}}"
|
||||
"message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }};\nIndex: [{{index .group_values 1}}]({{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22});\nMax Shard Storage：{{.result_value | format_bytes 2}};\n{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -1525,7 +1525,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-06-16T04:11:10.242061032Z",
|
||||
|
|
@ -1537,8 +1537,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Max Shard Storage:* {{.result_value | format_bytes 2}}\"\n },\n \n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Max Shard Storage:* {{.result_value | format_bytes 2}}\"\n },\n \n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -1564,7 +1564,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/cb34sfl6psfiqtovhpt4
|
|||
"creator": {
|
||||
"name": "$[[USERNAME]]",
|
||||
"id": "$[[USER_ID]]"
|
||||
},
|
||||
},
|
||||
"resource": {
|
||||
"resource_id": "$[[RESOURCE_ID]]",
|
||||
"resource_name": "$[[RESOURCE_NAME]]",
|
||||
|
|
@ -1613,7 +1613,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/cb34sfl6psfiqtovhpt4
|
|||
"format_type": "ratio",
|
||||
"expression": "(max(payload.elasticsearch.index_stats.primaries.docs.deleted)/(max(payload.elasticsearch.index_stats.primaries.docs.deleted)+max(payload.elasticsearch.index_stats.primaries.docs.count)))*100",
|
||||
"title": "Too Many Deleted Documents (>30%)",
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nIndex:{{index .group_values 0}}; \nRatio of Deleted Documents:{{.result_value}};\n{{end}}"
|
||||
"message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex:{{index .group_values 0}}; \nRatio of Deleted Documents:{{.result_value}};\n{{end}}"
|
||||
},
|
||||
"conditions": {
|
||||
"operator": "any",
|
||||
|
|
@ -1637,7 +1637,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/cb34sfl6psfiqtovhpt4
|
|||
]
|
||||
},
|
||||
"channels": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"normal": [
|
||||
{
|
||||
"created": "2022-06-16T04:11:10.242061032Z",
|
||||
|
|
@ -1649,8 +1649,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/cb34sfl6psfiqtovhpt4
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "${SLACK_WEBHOOK_ENDPOINT}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Deleted:* {{.result_value | to_fixed 2}}%\"\n },\n \n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
"url": "{{.env.SLACK_WEBHOOK_ENDPOINT}}",
|
||||
"body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Deleted:* {{.result_value | to_fixed 2}}%\"\n },\n \n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
|
|||
Loading…
Reference in New Issue