From c8aab5c56697961469f0c1b89a0f2521b221ec94 Mon Sep 17 00:00:00 2001 From: liugq Date: Thu, 10 Aug 2023 18:09:11 +0800 Subject: [PATCH] update alerting rule, channel template --- config/initialization.tpl | 1342 +++++++++++++++++----------------- config/initialization_v5.tpl | 1337 +++++++++++++++++---------------- config/initialization_v6.tpl | 1337 +++++++++++++++++---------------- 3 files changed, 2001 insertions(+), 2015 deletions(-) diff --git a/config/initialization.tpl b/config/initialization.tpl index c925328d..a084761b 100644 --- a/config/initialization.tpl +++ b/config/initialization.tpl @@ -603,65 +603,29 @@ PUT $[[INDEX_PREFIX]]activities-00001 #alerting channel #The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]channel/_doc/cj865st3q95rega919ig -{ - "id": "cj865st3q95rega919ig", - "created": "2023-08-07T11:20:19.223545026+08:00", - "updated": "2023-08-08T18:42:26.506499014+08:00", - "name": "[Alerting] Discord", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.DISCORD_WEBHOOK_ENDPOINT}}", - "body": "{\n \"content\": \"Hello Alerting\"\n}" - }, - "sub_type": "discord", - "enabled": false -} -POST $[[INDEX_PREFIX]]channel/_doc/cj86l0l3q95rrpfea6ug -{ - "id": "cj86l0l3q95rrpfea6ug", - "created": "2023-08-07T11:52:34.192522006+08:00", - "updated": "2023-08-08T18:42:30.162079286+08:00", - "name": "[Recovery] Discord\t", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.DISCORD_WEBHOOK_ENDPOINT}}", - "body": "{\n\n}" - }, - "sub_type": "discord", - "enabled": false -} POST $[[INDEX_PREFIX]]channel/_doc/cgnb2nt3q95nmusjl65g { - "id": "cgnb2nt3q95nmusjl65g", - "created": "2023-04-06T11:47:43.104108279Z", - "updated": "2023-08-08T22:19:08.601341574+08:00", - "name": "[Alerting] Slack Notification", - "type": "webhook", - "webhook": { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-09T22:39:50.494915568+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { "header_params": { "Content-type": "application/json" }, "method": "POST", "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.trigger_at | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Cluster:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" - }, - "sub_type": "slack", - "enabled": false + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing !*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.trigger_at | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Cluster:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": false } POST $[[INDEX_PREFIX]]channel/_doc/cj8bq8d3q95ogankugqg { "id": "cj8bq8d3q95ogankugqg", "created": "2023-08-07T17:45:05.534408059+08:00", - "updated": "2023-08-08T19:26:34.009668892+08:00", + "updated": "2023-08-09T22:39:56.489567891+08:00", "name": "[Recovery] Slack Notification", "type": "webhook", "webhook": { @@ -670,16 +634,34 @@ POST $[[INDEX_PREFIX]]channel/_doc/cj8bq8d3q95ogankugqg }, "method": "POST", "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*:rainbow: Alert [{{.rule_name}}] Resolved*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*ResolveAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Target:* {{.resource_name}}-{{.objects}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.trigger_at | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Duration:* {{.duration}}\"\n }\n },\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n }\n ]\n}" + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*ResolveAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Target:* {{.resource_name}}-{{.objects}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.trigger_at | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Duration:* {{.duration}}\"\n }\n },\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n }\n ]\n}" }, "sub_type": "slack", "enabled": false } +POST $[[INDEX_PREFIX]]channel/_doc/cgnb2kt3q95nmusjl64g +{ + "id": "cgnb2kt3q95nmusjl64g", + "created": "2023-04-06T11:47:31.161587662Z", + "updated": "2023-08-09T22:39:51.540172306+08:00", + "name": "[Alerting] Wechat Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.WECOM_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"**[ INFINI Platform Alerting ]**\\nšŸ”„ Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n**{{.title}}**\\nPriority: {{.priority}}\\n\\nEventID: {{.event_id}}\\n\\nTarget: {{.resource_name}}-{{.objects}}\\n\\nTriggerAt: {{.trigger_at | datetime}}\\n{{.message}}\"\n }\n}" + }, + "sub_type": "wechat", + "enabled": false +} POST $[[INDEX_PREFIX]]channel/_doc/cgiospt3q95q49k3u00g { "id": "cgiospt3q95q49k3u00g", "created": "2023-03-30T13:28:07.531263747Z", - "updated": "2023-08-08T22:19:07.545051029+08:00", + "updated": "2023-08-09T22:39:52.356059486+08:00", "name": "[Alerting] DingTalk Notification", "type": "webhook", "webhook": { @@ -688,16 +670,52 @@ POST $[[INDEX_PREFIX]]channel/_doc/cgiospt3q95q49k3u00g }, "method": "POST", "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.com/img/email/alert-header.png)\\n\\nšŸ”„ **{{.title}}**\\n\\nIncident [{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n\\nPriority: {{.priority}}\\n\\nEventID: {{.event_id}}\\n\\nTarget: {{.resource_name}}-{{.objects}}\\n\\nTriggerAt: {{.trigger_at | datetime}}\\n\\n---\\n\\n{{.message}}\"\n }\n}" + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.com/img/email/alert-header.png)\\n\\nšŸ”„ Incident [{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n\\n**{{.title}}**\\n\\nPriority: {{.priority}}\\n\\nEventID: {{.event_id}}\\n\\nTarget: {{.resource_name}}-{{.objects}}\\n\\nTriggerAt: {{.trigger_at | datetime}}\\n\\n---\\n\\n{{.message}}\"\n }\n}" }, "sub_type": "dingtalk", "enabled": false } +POST $[[INDEX_PREFIX]]channel/_doc/cj8ctat3q95l9ebbntlg +{ + "id": "cj8ctat3q95l9ebbntlg", + "created": "2023-08-07T18:59:55.28732241+08:00", + "updated": "2023-08-09T22:39:58.967970184+08:00", + "name": "[Recovery] DingTalk Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.com/img/email/recovery-header.png)\\n\\n**{{.title}}**\\n\\n{{.message}}\\n\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n }\n}\n" + }, + "sub_type": "dingtalk", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/_doc/cj8e9gt3q95gsdbb0170 +{ + "id": "cj8e9gt3q95gsdbb0170", + "created": "2023-08-07T20:34:11.998953512+08:00", + "updated": "2023-08-09T22:40:04.665871275+08:00", + "name": "[Recovery] Wechat Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.WECOM_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"**[ INFINI Platform Alerting ]**\\n**{{.title}}**\\n\\n{{.message}}\\n\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n }\n}\n" + }, + "sub_type": "wechat", + "enabled": false +} POST $[[INDEX_PREFIX]]channel/_doc/cgnb2r53q95nmusjl6vg { "id": "cgnb2r53q95nmusjl6vg", "created": "2023-04-06T11:47:56.652637309Z", - "updated": "2023-08-08T19:49:20.312590885+08:00", + "updated": "2023-08-10T12:04:08.046781556+08:00", "name": "[Alerting] Email Notification", "type": "email", "sub_type": "email", @@ -709,105 +727,15 @@ POST $[[INDEX_PREFIX]]channel/_doc/cgnb2r53q95nmusjl6vg }, "subject": "[INFINI Platform Alerting] šŸ”„ {{.title}}", "body": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n
\n \"email-header\"\n
\n
\n \n \n \n \n \n \n
\n
\n
\n {{.title}}\n

\n \n

Priority: {{.priority}}

\n

EventID: {{.event_id}}

\n

Target: {{.resource_name}}-{{.objects}}

\n

TriggerAt: {{.trigger_at | datetime}}

\n {{.message | md_to_html}}\n
\n

\n \n \n View Detail\n \n

\n \n \n \n

\n \n

\n \"INFINI\n
\n
\n \n
\n

\n \n \n
\n
\n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n
\n
\n \n \n \n
\n
\n \n
\n \n \n
\n
\n \n ", - "content_type": "text/html" + "content_type": "text/html" }, "enabled": false } -POST $[[INDEX_PREFIX]]channel/_doc/ch1os6t3q95lk6lepkq0 -{ - "id": "ch1os6t3q95lk6lepkq0", - "created": "2023-04-22T07:34:51.848540351Z", - "updated": "2023-08-09T09:29:26.412223281+08:00", - "name": "[Alerting] Feishu Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI Platform Alerting ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"{{if eq .priority \"critical\"}}red{{else if eq .priority \"high\"}}orange{{else if eq .priority \"medium\"}}yellow{{else if eq .priority \"low\"}}grey{{else}}blue{{end}}\"\n },\n \"elements\": [{\n \"tag\": \"markdown\",\n \"content\": \"šŸ”„ Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n **{{.title}}**\\nPriority: {{.priority}}\\nEventID: {{.event_id}}\\nTarget: {{.resource_name}}-{{.objects}}\\nTriggerAt: {{.trigger_at | datetime}}\"\n },{\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n }\n ]\n}\n}" - }, - "sub_type": "feishu", - "enabled": false -} -POST $[[INDEX_PREFIX]]channel/_doc/cj8e9s53q95gsdbb054g -{ - "id": "cj8e9s53q95gsdbb054g", - "created": "2023-08-07T20:34:56.334695598+08:00", - "updated": "2023-08-08T21:34:50.261294305+08:00", - "name": "[Recovery] Feishu Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI Platform Alerting ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"green\"\n },\n \"elements\": [\n {\n \"tag\": \"markdown\",\n \"content\": \"šŸŒˆ **{{.title}}**\"\n },\n {\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n }\n ]\n }\n}" - }, - "sub_type": "feishu", - "enabled": false -} -POST $[[INDEX_PREFIX]]channel/_doc/cj8ctat3q95l9ebbntlg - { - "id": "cj8ctat3q95l9ebbntlg", - "created": "2023-08-07T18:59:55.28732241+08:00", - "updated": "2023-08-08T19:46:30.557046793+08:00", - "name": "[Recovery] DingTalk Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.com/img/email/recovery-header.png)\\n\\nšŸŒˆ **{{.title}}**\\n\\n{{.message}}\\n\\n---\\n\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n }\n}\n" - }, - "sub_type": "dingtalk", - "enabled": false -} -POST $[[INDEX_PREFIX]]channel/_doc/cgnb2kt3q95nmusjl64g -{ - "id": "cgnb2kt3q95nmusjl64g", - "created": "2023-04-06T11:47:31.161587662Z", - "updated": "2023-08-08T22:19:06.712911427+08:00", - "name": "[Alerting] Wechat Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.WECOM_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"**[ INFINI Platform Alerting ]**\\nšŸ”„ Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n**{{.title}}**\\n>Priority: {{.priority}}\\n\\n>EventID: {{.event_id}}\\n\\n>Target: {{.resource_name}}-{{.objects}}\\n\\n>TriggerAt: {{.trigger_at | datetime}}\\n\\n{{.message}}\"\n }\n}" - }, - "sub_type": "wechat", - "enabled": false -} -POST $[[INDEX_PREFIX]]channel/_doc/cj8e9gt3q95gsdbb0170 -{ - "id": "cj8e9gt3q95gsdbb0170", - "created": "2023-08-07T20:34:11.998953512+08:00", - "updated": "2023-08-08T19:47:08.270014715+08:00", - "name": "[Recovery] Wechat Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.WECOM_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"**[ INFINI Platform Alerting ]**\\nšŸŒˆ **{{.title}}**\\n\\n{{.message}}\\n\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n }\n}\n" - }, - "sub_type": "wechat", - "enabled": false -} POST $[[INDEX_PREFIX]]channel/_doc/cj8atf53q95lhahebg8g { "id": "cj8atf53q95lhahebg8g", "created": "2023-08-07T16:43:40.062389175+08:00", - "updated": "2023-08-08T19:50:15.803258835+08:00", + "updated": "2023-08-10T12:04:42.842628127+08:00", "name": "[Recovery] Email Notification", "type": "email", "sub_type": "email", @@ -817,20 +745,93 @@ POST $[[INDEX_PREFIX]]channel/_doc/cj8atf53q95lhahebg8g "to": [], "cc": [] }, - "subject": "[INFINI Platform Alerting] šŸŒˆ {{.title}}", - "body": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n
\n \"email-header\"\n
\n
\n \n \n \n \n \n \n
\n
\n
\n šŸŒˆ {{.title}}\n

\n {{.message | md_to_html}}\n
\n

\n \n \n View Detail\n \n

\n \n \n \n

\n \n

\n \"INFINI\n
\n
\n \n
\n

\n \n \n
\n
\n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n
\n
\n \n \n \n
\n
\n \n
\n \n \n
\n
\n \n ", + "subject": "[INFINI Platform Alerting] {{.title}}", + "body": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n
\n \"email-header\"\n
\n
\n \n \n \n \n \n \n
\n
\n
\n {{.title}}\n

\n {{.message | md_to_html}}\n
\n

\n \n \n View Detail\n \n

\n \n \n \n

\n \n

\n \"INFINI\n
\n
\n \n
\n

\n \n \n
\n
\n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n
\n
\n \n \n \n
\n
\n \n
\n \n \n
\n
\n \n ", "content_type": "text/html" }, "enabled": false } +POST $[[INDEX_PREFIX]]channel/_doc/ch1os6t3q95lk6lepkq0 +{ + "id": "ch1os6t3q95lk6lepkq0", + "created": "2023-04-22T07:34:51.848540351Z", + "updated": "2023-08-10T17:18:38.592432088+08:00", + "name": "[Alerting] Feishu Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI Platform Alerting ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"{{if eq .priority \"critical\"}}red{{else if eq .priority \"high\"}}orange{{else if eq .priority \"medium\"}}yellow{{else if eq .priority \"low\"}}grey{{else}}blue{{end}}\"\n },\n \"elements\": [{\n \"tag\": \"markdown\",\n \"content\": \"šŸ”„ Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n **{{.title}}**\\nPriority: {{.priority}}\\nEventID: {{.event_id}}\\nTarget: {{.resource_name}}-{{.objects}}\\nTriggerAt: {{.trigger_at | datetime}}\"\n },{\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n }\n ]\n}\n}" + }, + "sub_type": "feishu", + "enabled": false + } +POST $[[INDEX_PREFIX]]channel/_doc/cj8e9s53q95gsdbb054g +{ + "id": "cj8e9s53q95gsdbb054g", + "created": "2023-08-07T20:34:56.334695598+08:00", + "updated": "2023-08-10T17:18:36.035896482+08:00", + "name": "[Recovery] Feishu Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI Platform Alerting ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"green\"\n },\n \"elements\": [\n {\n \"tag\": \"markdown\",\n \"content\": \"šŸŒˆ **{{.title}}**\"\n },\n {\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n },\n {\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"[View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n }\n ]\n }\n}" + }, + "sub_type": "feishu", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/_doc/cj865st3q95rega919ig +{ + "id": "cj865st3q95rega919ig", + "created": "2023-08-07T11:20:19.223545026+08:00", + "updated": "2023-08-10T17:18:41.92016786+08:00", + "name": "[Alerting] Discord Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DISCORD_WEBHOOK_ENDPOINT}}", + "body": "{\"content\": \"**[ INFINI Platform Alerting ]**\\nšŸ”„ Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n**{{.title}}**\\n\\nPriority: {{.priority}}\\nEventID: {{.event_id}}\\nTarget: {{.resource_name}}-{{.objects}}\\nTriggerAt: {{.trigger_at | datetime}}\\n{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"}" + }, + "sub_type": "discord", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/_doc/cj86l0l3q95rrpfea6ug +{ + "id": "cj86l0l3q95rrpfea6ug", + "created": "2023-08-07T11:52:34.192522006+08:00", + "updated": "2023-08-10T17:18:44.422687739+08:00", + "name": "[Recovery] Discord Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DISCORD_WEBHOOK_ENDPOINT}}", + "body": "{\n \"content\": \"**[ INFINI Platform Alerting ]**\\nšŸŒˆ **{{.title}}**\\n\\n{{.message | str_replace \"\\n\" \"\\\\n\" }}\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n}" + }, + "sub_type": "discord", + "enabled": false +} #alerting -POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calgapp7h710dpnpbeb6 +#The `id` value is consistent with the `_id` value +POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnoaps0 { - "id": "builtin-calgapp7h710dpnpbeb6", - "created": "2022-06-16T10:26:47.360988761Z", - "updated": "2023-08-09T09:44:58.584645596+08:00", - "name": "Shard Storage >= 55G", + "id": "builtin-cal8n7p7h710dpnoaps0", + "created": "2022-06-16T01:47:11.326727124Z", + "updated": "2023-08-09T22:39:43.98598502+08:00", + "name": "Cluster Health Change to Red", "enabled": true, "resource": { "resource_id": "$[[RESOURCE_ID]]", @@ -841,10 +842,21 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calgapp7h710dpnpbeb6 ], "filter": {}, "raw_filter": { - "range": { - "payload.elasticsearch.index_stats.shard_info.store_in_bytes": { - "gte": 59055800320 - } + "bool": { + "must": [ + { + "match": { + "payload.elasticsearch.cluster_health.status": "red" + } + }, + { + "term": { + "metadata.name": { + "value": "cluster_health" + } + } + } + ] } }, "time_field": "timestamp", @@ -858,25 +870,21 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calgapp7h710dpnpbeb6 { "field": "metadata.labels.cluster_id", "limit": 5 - }, - { - "field": "metadata.labels.index_name", - "limit": 500 } ], "formula": "a", "items": [ { "name": "a", - "field": "payload.elasticsearch.index_stats.shard_info.store_in_bytes", - "statistic": "max" + "field": "payload.elasticsearch.cluster_health.status", + "statistic": "count" } ], - "format_type": "bytes", + "format_type": "num", "bucket_label": { "enabled": false }, - "expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)" + "expression": "count(payload.elasticsearch.cluster_health.status)" }, "conditions": { "operator": "any", @@ -885,40 +893,39 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calgapp7h710dpnpbeb6 "minimum_period_match": 1, "operator": "gte", "values": [ - "59055800320" + "1" ], - "priority": "high" + "priority": "critical" } ] }, "notification_config": { "enabled": true, - "title": "Shard Storage >55GB in ({{len .results}} indices in total)", - "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}), Max Shard Storage: {{.result_value | format_bytes 2}}\n{{end}}", + "title": "Health of Clusters ({{len .results}} clusters in total) Changed to Red", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nCluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) is Red now\n{{end}}", "normal": [ { "id": "cgnb2nt3q95nmusjl65g", "created": "2023-04-06T11:47:43.104108279Z", - "updated": "2023-08-07T14:02:53.734855705+08:00", + "updated": "2023-08-07T15:02:17.165625799+08:00", "name": "[Alerting] Slack Notification", "type": "webhook", "webhook": { "header_params": { - "Content-type": "application/json" + "Content-Type": "application/json" }, "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Max shard storage: {{.result_value | format_bytes 2}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Cluster:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}> is Red now\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" }, "sub_type": "slack", "enabled": true }, { - "id": "cj865st3q95rega919ig", + "id": "cgiospt3q95q49k3u00g", "enabled": true }, { - "id": "cgiospt3q95q49k3u00g", + "id": "cj865st3q95rega919ig", "enabled": true }, { @@ -942,8 +949,160 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calgapp7h710dpnpbeb6 }, "recovery_notification_config": { "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "- EventID: {{.event_id}}\n- Target: {{.resource_name}}-{{.objects}}\n- TriggerAt: {{.trigger_at}}\n- ResolveAt: {{.timestamp | datetime}}\n- Duration: {{.duration}}", + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calavvp7h710dpnp32r3 +{ + "id": "builtin-calavvp7h710dpnp32r3", + "created": "2022-06-16T04:22:23.001354546Z", + "updated": "2023-08-09T22:20:17.864619426+08:00", + "name": "Index Health Change to Red", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_index" + ], + "filter": {}, + "raw_filter": { + "match": { + "metadata.labels.health_status": "red" + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.cluster_id", + "limit": 50 + }, + { + "field": "metadata.index_name", + "limit": 1000 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "metadata.index_name", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(metadata.index_name)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Health of Indices ({{len .results}} indices in total) Changed to Red", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nIndex: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D) is Red now\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T15:17:26.18861218+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{ lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0)}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}> is Red now\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", "normal": [ { "id": "cj8bq8d3q95ogankugqg", @@ -980,11 +1139,163 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calgapp7h710dpnpbeb6 "id": "$[[USER_ID]]" } } +POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cbp20n2anisjmu4gehc5 +{ + "id": "builtin-cbp20n2anisjmu4gehc5", + "created": "2022-08-09T08:52:44.63345561Z", + "updated": "2023-08-09T22:11:45.679048697+08:00", + "name": "Elasticsearch node left cluster", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_node" + ], + "filter": {}, + "raw_filter": { + "match_phrase": { + "metadata.labels.status": "unavailable" + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.cluster_id", + "limit": 5 + }, + { + "field": "metadata.node_id", + "limit": 50 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "metadata.labels.status", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(metadata.labels.status)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "critical" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Elasticsearch node left cluster", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nNode: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Left: {{.result_value}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T10:42:17.686776304+08:00", + "name": "Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Left: {{.result_value}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cb34sfl6psfiqtovhpt4 { "id": "builtin-cb34sfl6psfiqtovhpt4", "created": "2022-07-07T03:08:46.297166036Z", - "updated": "2023-08-09T09:45:34.123901475+08:00", + "updated": "2023-08-09T22:38:41.764325087+08:00", "name": "Too Many Deleted Documents", "enabled": true, "resource": { @@ -1070,7 +1381,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cb34sfl6psfiqtovhpt4 "notification_config": { "enabled": true, "title": "Too Many Deleted Documents (>30%)", - "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), Deleted: {{.result_value | to_fixed 2}}%\n{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nIndex: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Deleted: {{.result_value | to_fixed 2}}%\n{{end}}", "normal": [ { "id": "cgnb2nt3q95nmusjl65g", @@ -1081,7 +1392,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cb34sfl6psfiqtovhpt4 "Content-type": "application/json" }, "method": "POST", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Deleted ratio: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{ lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0)}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Deleted ratio: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" }, "sub_type": "", "enabled": true @@ -1115,314 +1426,8 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cb34sfl6psfiqtovhpt4 }, "recovery_notification_config": { "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", - "normal": [ - { - "id": "cj8bq8d3q95ogankugqg", - "enabled": true - }, - { - "id": "cj8ctat3q95l9ebbntlg", - "enabled": true - }, - { - "id": "cj8atf53q95lhahebg8g", - "enabled": true - }, - { - "id": "cj8e9s53q95gsdbb054g", - "enabled": true - }, - { - "id": "cj8e9gt3q95gsdbb0170", - "enabled": true - }, - { - "id": "cj86l0l3q95rrpfea6ug", - "enabled": true - } - ], - "event_enabled": true - }, - "schedule": { - "interval": "1m" - }, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - } -} - -POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cbp20n2anisjmu4gehc5 -{ - "id": "builtin-cbp20n2anisjmu4gehc5", - "created": "2022-08-09T08:52:44.63345561Z", - "updated": "2023-08-09T09:43:37.945659792+08:00", - "name": "Elasticsearch node left cluster", - "enabled": true, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - ".infini_node" - ], - "filter": {}, - "raw_filter": { - "match_phrase": { - "metadata.labels.status": "unavailable" - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.cluster_id", - "limit": 5 - }, - { - "field": "metadata.node_id", - "limit": 50 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "metadata.labels.status", - "statistic": "count" - } - ], - "format_type": "num", - "bucket_label": { - "enabled": false - }, - "expression": "count(metadata.labels.status)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "1" - ], - "priority": "critical" - } - ] - }, - "notification_config": { - "enabled": true, - "title": "Elasticsearch node left cluster", - "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), Left: {{.result_value}}\n{{end}}", - "normal": [ - { - "id": "cgnb2nt3q95nmusjl65g", - "created": "2023-04-06T11:47:43.104108279Z", - "updated": "2023-08-07T10:42:17.686776304+08:00", - "name": "Slack Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-type": "application/json" - }, - "method": "POST", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Left: {{.result_value}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" - }, - "sub_type": "slack", - "enabled": true - }, - { - "id": "cgnb2r53q95nmusjl6vg", - "enabled": true - }, - { - "id": "ch1os6t3q95lk6lepkq0", - "enabled": true - }, - { - "id": "cgnb2kt3q95nmusjl64g", - "enabled": true - }, - { - "id": "cj865st3q95rega919ig", - "enabled": true - }, - { - "id": "cgiospt3q95q49k3u00g", - "enabled": true - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "recovery_notification_config": { - "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", - "normal": [ - { - "id": "cj8bq8d3q95ogankugqg", - "enabled": true - }, - { - "id": "cj8ctat3q95l9ebbntlg", - "enabled": true - }, - { - "id": "cj8e9s53q95gsdbb054g", - "enabled": true - }, - { - "id": "cj8e9gt3q95gsdbb0170", - "enabled": true - }, - { - "id": "cj86l0l3q95rrpfea6ug", - "enabled": true - }, - { - "id": "cj8atf53q95lhahebg8g", - "enabled": true - } - ], - "event_enabled": true - }, - "schedule": { - "interval": "1m" - }, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - } -} -POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calavvp7h710dpnp32r3 -{ - "id": "builtin-calavvp7h710dpnp32r3", - "created": "2022-06-16T04:22:23.001354546Z", - "updated": "2023-08-09T09:43:58.551403706+08:00", - "name": "Index Health Change to Red", - "enabled": true, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - ".infini_index" - ], - "filter": {}, - "raw_filter": { - "match": { - "metadata.labels.health_status": "red" - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.cluster_id", - "limit": 50 - }, - { - "field": "metadata.index_name", - "limit": 1000 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "metadata.index_name", - "statistic": "count" - } - ], - "format_type": "num", - "bucket_label": { - "enabled": false - }, - "expression": "count(metadata.index_name)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "1" - ], - "priority": "high" - } - ] - }, - "notification_config": { - "enabled": true, - "title": "Health of Indices ({{len .results}} indices in total) Changed to Red", - "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}) is Red now\n{{end}}", - "normal": [ - { - "id": "cgnb2nt3q95nmusjl65g", - "created": "2023-04-06T11:47:43.104108279Z", - "updated": "2023-08-07T15:17:26.18861218+08:00", - "name": "[Alerting] Slack Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}> is Red now\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" - }, - "sub_type": "slack", - "enabled": true - }, - { - "id": "cgiospt3q95q49k3u00g", - "enabled": true - }, - { - "id": "cj865st3q95rega919ig", - "enabled": true - }, - { - "id": "cgnb2r53q95nmusjl6vg", - "enabled": true - }, - { - "id": "ch1os6t3q95lk6lepkq0", - "enabled": true - }, - { - "id": "cgnb2kt3q95nmusjl64g", - "enabled": true - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "recovery_notification_config": { - "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", "normal": [ { "id": "cj8bq8d3q95ogankugqg", @@ -1463,7 +1468,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cbp2e4ianisjmu4giqs7 { "id": "builtin-cbp2e4ianisjmu4giqs7", "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2023-08-09T09:44:31.495696286+08:00", + "updated": "2023-08-09T22:39:15.339913317+08:00", "name": "Search latency is great than 500ms", "enabled": true, "resource": { @@ -1563,7 +1568,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cbp2e4ianisjmu4giqs7 "notification_config": { "enabled": true, "title": "Search latency is great than 500ms", - "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}), Latency: {{.result_value | to_fixed 2}}ms\n{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nIndex: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Latency: {{.result_value | to_fixed 2}}ms\n{{end}}", "normal": [ { "id": "cgnb2nt3q95nmusjl65g", @@ -1576,8 +1581,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cbp2e4ianisjmu4giqs7 "Content-type": "application/json" }, "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "\n{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Search latency: {{.result_value | to_fixed 2}}ms\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + "body": "\n{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{ lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0)}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Search latency: {{.result_value | to_fixed 2}}ms\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" }, "sub_type": "slack", "enabled": true @@ -1611,8 +1615,8 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cbp2e4ianisjmu4giqs7 }, "recovery_notification_config": { "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", "normal": [ { "id": "cj8bq8d3q95ogankugqg", @@ -1653,7 +1657,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calaqnh7h710dpnp2bm8 { "id": "builtin-calaqnh7h710dpnp2bm8", "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2023-08-09T09:46:34.428920151+08:00", + "updated": "2023-08-09T22:38:55.677122718+08:00", "name": "JVM utilization is Too High", "enabled": true, "resource": { @@ -1747,7 +1751,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calaqnh7h710dpnp2bm8 "notification_config": { "enabled": true, "title": "JVM Usage of Nodes ({{len .results}} nodes in total) >= {{.first_threshold}}%", - "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/{{index .group_values 0}}/nodes/{{index .group_values 1}}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), JVM Usage: {{.result_value | to_fixed 2}}%\n{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nNode: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), JVM Usage: {{.result_value | to_fixed 2}}%\n{{end}}", "normal": [ { "id": "cgnb2nt3q95nmusjl65g", @@ -1760,7 +1764,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calaqnh7h710dpnp2bm8 "Content-type": "application/json" }, "method": "POST", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, JVM Usage: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, JVM Usage: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" }, "sub_type": "slack", "enabled": true @@ -1794,8 +1798,8 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calaqnh7h710dpnp2bm8 }, "recovery_notification_config": { "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", "normal": [ { "id": "cj8bq8d3q95ogankugqg", @@ -1836,7 +1840,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calakp97h710dpnp1fa2 { "id": "builtin-calakp97h710dpnp1fa2", "created": "2022-06-16T03:58:29.437447113Z", - "updated": "2023-08-09T09:42:57.901272952+08:00", + "updated": "2023-08-09T22:33:25.692835454+08:00", "name": "CPU utilization is Too High", "enabled": true, "resource": { @@ -1905,7 +1909,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calakp97h710dpnp1fa2 "minimum_period_match": 1, "operator": "gte", "values": [ - "85" + "80" ], "priority": "low" }, @@ -1930,7 +1934,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calakp97h710dpnp1fa2 "notification_config": { "enabled": true, "title": "CPU Usage of Nodes ({{len .results}} nodes in total) >= {{.first_threshold}}%", - "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), CPU Usage: {{.result_value | to_fixed 2}}%\n{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nNode: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), CPU Usage: {{.result_value | to_fixed 2}}%\n{{end}}", "normal": [ { "id": "cgnb2nt3q95nmusjl65g", @@ -1943,8 +1947,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calakp97h710dpnp1fa2 "Content-type": "application/json" }, "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, CPU Usage: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, CPU Usage: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" }, "sub_type": "slack", "enabled": true @@ -1978,8 +1981,162 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calakp97h710dpnp1fa2 }, "recovery_notification_config": { "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calgapp7h710dpnpbeb6 +{ + "id": "builtin-calgapp7h710dpnpbeb6", + "created": "2022-06-16T10:26:47.360988761Z", + "updated": "2023-08-09T22:37:44.038127695+08:00", + "name": "Shard Storage >= 55G", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "range": { + "payload.elasticsearch.index_stats.shard_info.store_in_bytes": { + "gte": 59055800320 + } + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.index_name", + "limit": 500 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.index_stats.shard_info.store_in_bytes", + "statistic": "max" + } + ], + "format_type": "bytes", + "bucket_label": { + "enabled": false + }, + "expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "59055800320" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Shard Storage >55GB in ({{len .results}} indices in total)", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nIndex: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Max Shard Storage: {{.result_value | format_bytes 2}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T14:02:53.734855705+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{ lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0)}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Max shard storage: {{.result_value | format_bytes 2}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "- EventID: {{.event_id}}\n- Target: {{.resource_name}}-{{.objects}}\n- TriggerAt: {{.trigger_at | datetime}}\n- ResolveAt: {{.timestamp | datetime}}\n- Duration: {{.duration}}", "normal": [ { "id": "cj8bq8d3q95ogankugqg", @@ -2020,7 +2177,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnogps1 { "id": "builtin-cal8n7p7h710dpnogps1", "created": "2022-06-16T03:11:01.445958361Z", - "updated": "2023-08-09T09:43:16.31964237+08:00", + "updated": "2023-08-10T17:16:34.900352415+08:00", "name": "Disk utilization is Too High", "enabled": true, "resource": { @@ -2119,7 +2276,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnogps1 "notification_config": { "enabled": true, "title": "Disk Usage of Nodes ({{len .results}} nodes in total) >= {{.first_threshold}}%", - "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), Usage: {{.result_value | to_fixed 2}}% / Free: {{.relation_values.b | format_bytes 2}}\n{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nNode: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Usage: {{.result_value | to_fixed 2}}% / Free: {{.relation_values.b | format_bytes 2}}\n{{end}}", "normal": [ { "id": "cgnb2nt3q95nmusjl65g", @@ -2130,7 +2287,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnogps1 "Content-type": "application/json" }, "method": "POST", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Disk Usage: {{.result_value | to_fixed 2}}%, Free: {{.relation_values.b | format_bytes 2}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n },\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Document\" \n },\n \"style\": \"primary\",\n \"url\": \"https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-cluster.html#disk-based-shard-allocation\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Disk Usage: {{.result_value | to_fixed 2}}%, Free: {{.relation_values.b | format_bytes 2}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n },\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Document\" \n },\n \"style\": \"primary\",\n \"url\": \"https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-cluster.html#disk-based-shard-allocation\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" }, "sub_type": "", "enabled": true @@ -2164,175 +2321,9 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnogps1 }, "recovery_notification_config": { "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", - "normal": [ - { - "id": "cj8bq8d3q95ogankugqg", - "enabled": true - }, - { - "id": "cj8ctat3q95l9ebbntlg", - "enabled": true - }, - { - "id": "cj8atf53q95lhahebg8g", - "enabled": true - }, - { - "id": "cj8e9s53q95gsdbb054g", - "enabled": true - }, - { - "id": "cj8e9gt3q95gsdbb0170", - "enabled": true - }, - { - "id": "cj86l0l3q95rrpfea6ug", - "enabled": true - } - ], - "event_enabled": true - }, - "schedule": { - "interval": "1m" - }, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - } -} -POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnoaps0 -{ - "id": "builtin-cal8n7p7h710dpnoaps0", - "created": "2022-06-16T01:47:11.326727124Z", - "updated": "2023-08-09T09:50:05.833535441+08:00", - "name": "Cluster Health Change to Red", - "enabled": true, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - ".infini_metrics*" - ], - "filter": {}, - "raw_filter": { - "bool": { - "must": [ - { - "match": { - "payload.elasticsearch.cluster_health.status": "red" - } - }, - { - "term": { - "metadata.name": { - "value": "cluster_health" - } - } - } - ] - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.cluster_health.status", - "statistic": "count" - } - ], - "format_type": "num", - "bucket_label": { - "enabled": false - }, - "expression": "count(payload.elasticsearch.cluster_health.status)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "1" - ], - "priority": "critical" - } - ] - }, - "notification_config": { - "enabled": true, - "title": "Health of Clusters ({{len .results}} clusters in total) Changed to Red", - "message": "{{range .results}}\nCluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/) is Red now\n{{end}}", - "normal": [ - { - "id": "cgnb2nt3q95nmusjl65g", - "created": "2023-04-06T11:47:43.104108279Z", - "updated": "2023-08-07T15:02:17.165625799+08:00", - "name": "[Alerting] Slack Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Cluster:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}> is Red now\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" - }, - "sub_type": "slack", - "enabled": true - }, - { - "id": "cgiospt3q95q49k3u00g", - "enabled": true - }, - { - "id": "cj865st3q95rega919ig", - "enabled": true - }, - { - "id": "cgnb2r53q95nmusjl6vg", - "enabled": true - }, - { - "id": "ch1os6t3q95lk6lepkq0", - "enabled": true - }, - { - "id": "cgnb2kt3q95nmusjl64g", - "enabled": true - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "recovery_notification_config": { - "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", + "title": "šŸŒˆ [{{.rule_name}}] Resolved", "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", "normal": [ - { - "id": "cj8atf53q95lhahebg8g", - "enabled": true - }, { "id": "cj8bq8d3q95ogankugqg", "enabled": true @@ -2341,6 +2332,10 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnoaps0 "id": "cj8ctat3q95l9ebbntlg", "enabled": true }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, { "id": "cj8e9s53q95gsdbb054g", "enabled": true @@ -2365,6 +2360,7 @@ POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnoaps0 } } + #The `id` value is consistent with the `_id` value POST $[[INDEX_PREFIX]]view/_doc/cb34sfl6psfiqtovhpt4 { diff --git a/config/initialization_v5.tpl b/config/initialization_v5.tpl index e51cf740..b9bf131d 100644 --- a/config/initialization_v5.tpl +++ b/config/initialization_v5.tpl @@ -548,65 +548,29 @@ PUT $[[INDEX_PREFIX]]activities-00001 #alerting channel #The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]channel/doc/cj865st3q95rega919ig -{ - "id": "cj865st3q95rega919ig", - "created": "2023-08-07T11:20:19.223545026+08:00", - "updated": "2023-08-08T18:42:26.506499014+08:00", - "name": "[Alerting] Discord", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.DISCORD_WEBHOOK_ENDPOINT}}", - "body": "{\n \"content\": \"Hello Alerting\"\n}" - }, - "sub_type": "discord", - "enabled": false -} -POST $[[INDEX_PREFIX]]channel/doc/cj86l0l3q95rrpfea6ug -{ - "id": "cj86l0l3q95rrpfea6ug", - "created": "2023-08-07T11:52:34.192522006+08:00", - "updated": "2023-08-08T18:42:30.162079286+08:00", - "name": "[Recovery] Discord\t", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.DISCORD_WEBHOOK_ENDPOINT}}", - "body": "{\n\n}" - }, - "sub_type": "discord", - "enabled": false -} POST $[[INDEX_PREFIX]]channel/doc/cgnb2nt3q95nmusjl65g { - "id": "cgnb2nt3q95nmusjl65g", - "created": "2023-04-06T11:47:43.104108279Z", - "updated": "2023-08-08T22:19:08.601341574+08:00", - "name": "[Alerting] Slack Notification", - "type": "webhook", - "webhook": { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-09T22:39:50.494915568+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { "header_params": { "Content-type": "application/json" }, "method": "POST", "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing !*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.trigger_at | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Cluster:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" - }, - "sub_type": "slack", - "enabled": false + }, + "sub_type": "slack", + "enabled": false } POST $[[INDEX_PREFIX]]channel/doc/cj8bq8d3q95ogankugqg { "id": "cj8bq8d3q95ogankugqg", "created": "2023-08-07T17:45:05.534408059+08:00", - "updated": "2023-08-08T19:26:34.009668892+08:00", + "updated": "2023-08-09T22:39:56.489567891+08:00", "name": "[Recovery] Slack Notification", "type": "webhook", "webhook": { @@ -615,16 +579,34 @@ POST $[[INDEX_PREFIX]]channel/doc/cj8bq8d3q95ogankugqg }, "method": "POST", "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*:rainbow: Alert [{{.rule_name}}] Resolved*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*ResolveAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Target:* {{.resource_name}}-{{.objects}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.trigger_at | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Duration:* {{.duration}}\"\n }\n },\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n }\n ]\n}" + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*ResolveAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Target:* {{.resource_name}}-{{.objects}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.trigger_at | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Duration:* {{.duration}}\"\n }\n },\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n }\n ]\n}" }, "sub_type": "slack", "enabled": false } +POST $[[INDEX_PREFIX]]channel/doc/cgnb2kt3q95nmusjl64g +{ + "id": "cgnb2kt3q95nmusjl64g", + "created": "2023-04-06T11:47:31.161587662Z", + "updated": "2023-08-09T22:39:51.540172306+08:00", + "name": "[Alerting] Wechat Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.WECOM_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"**[ INFINI Platform Alerting ]**\\nšŸ”„ Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n**{{.title}}**\\nPriority: {{.priority}}\\n\\nEventID: {{.event_id}}\\n\\nTarget: {{.resource_name}}-{{.objects}}\\n\\nTriggerAt: {{.trigger_at | datetime}}\\n{{.message}}\"\n }\n}" + }, + "sub_type": "wechat", + "enabled": false +} POST $[[INDEX_PREFIX]]channel/doc/cgiospt3q95q49k3u00g { "id": "cgiospt3q95q49k3u00g", "created": "2023-03-30T13:28:07.531263747Z", - "updated": "2023-08-08T22:19:07.545051029+08:00", + "updated": "2023-08-09T22:39:52.356059486+08:00", "name": "[Alerting] DingTalk Notification", "type": "webhook", "webhook": { @@ -633,16 +615,52 @@ POST $[[INDEX_PREFIX]]channel/doc/cgiospt3q95q49k3u00g }, "method": "POST", "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.com/img/email/alert-header.png)\\n\\nšŸ”„ **{{.title}}**\\n\\nIncident [{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing !\\n\\nPriority: {{.priority}}\\n\\nEventID: {{.event_id}}\\n\\nTarget: {{.resource_name}}-{{.objects}}\\n\\nTriggerAt: {{.trigger_at | datetime}}\\n\\n---\\n\\n{{.message}}\"\n }\n}" + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.com/img/email/alert-header.png)\\n\\nšŸ”„ Incident [{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n\\n**{{.title}}**\\n\\nPriority: {{.priority}}\\n\\nEventID: {{.event_id}}\\n\\nTarget: {{.resource_name}}-{{.objects}}\\n\\nTriggerAt: {{.trigger_at | datetime}}\\n\\n---\\n\\n{{.message}}\"\n }\n}" }, "sub_type": "dingtalk", "enabled": false } +POST $[[INDEX_PREFIX]]channel/doc/cj8ctat3q95l9ebbntlg +{ + "id": "cj8ctat3q95l9ebbntlg", + "created": "2023-08-07T18:59:55.28732241+08:00", + "updated": "2023-08-09T22:39:58.967970184+08:00", + "name": "[Recovery] DingTalk Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.com/img/email/recovery-header.png)\\n\\n**{{.title}}**\\n\\n{{.message}}\\n\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n }\n}\n" + }, + "sub_type": "dingtalk", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cj8e9gt3q95gsdbb0170 +{ + "id": "cj8e9gt3q95gsdbb0170", + "created": "2023-08-07T20:34:11.998953512+08:00", + "updated": "2023-08-09T22:40:04.665871275+08:00", + "name": "[Recovery] Wechat Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.WECOM_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"**[ INFINI Platform Alerting ]**\\n**{{.title}}**\\n\\n{{.message}}\\n\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n }\n}\n" + }, + "sub_type": "wechat", + "enabled": false +} POST $[[INDEX_PREFIX]]channel/doc/cgnb2r53q95nmusjl6vg { "id": "cgnb2r53q95nmusjl6vg", "created": "2023-04-06T11:47:56.652637309Z", - "updated": "2023-08-08T19:49:20.312590885+08:00", + "updated": "2023-08-10T12:04:08.046781556+08:00", "name": "[Alerting] Email Notification", "type": "email", "sub_type": "email", @@ -654,105 +672,15 @@ POST $[[INDEX_PREFIX]]channel/doc/cgnb2r53q95nmusjl6vg }, "subject": "[INFINI Platform Alerting] šŸ”„ {{.title}}", "body": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n
\n \"email-header\"\n
\n
\n \n \n \n \n \n \n
\n
\n
\n {{.title}}\n

\n \n

Priority: {{.priority}}

\n

EventID: {{.event_id}}

\n

Target: {{.resource_name}}-{{.objects}}

\n

TriggerAt: {{.trigger_at | datetime}}

\n {{.message | md_to_html}}\n
\n

\n \n \n View Detail\n \n

\n \n \n \n

\n \n

\n \"INFINI\n
\n
\n \n
\n

\n \n \n
\n
\n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n
\n
\n \n \n \n
\n
\n \n
\n \n \n
\n
\n \n ", - "content_type": "text/html" + "content_type": "text/html" }, "enabled": false } -POST $[[INDEX_PREFIX]]channel/doc/ch1os6t3q95lk6lepkq0 -{ - "id": "ch1os6t3q95lk6lepkq0", - "created": "2023-04-22T07:34:51.848540351Z", - "updated": "2023-08-09T09:29:26.412223281+08:00", - "name": "[Alerting] Feishu Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI Platform Alerting ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"{{if eq .priority \"critical\"}}red{{else if eq .priority \"high\"}}orange{{else if eq .priority \"medium\"}}yellow{{else if eq .priority \"low\"}}grey{{else}}blue{{end}}\"\n },\n \"elements\": [{\n \"tag\": \"markdown\",\n \"content\": \"šŸ”„ Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing ! \\n **{{.title}}**\\nPriority: {{.priority}}\\nEventID: {{.event_id}}\\nTarget: {{.resource_name}}-{{.objects}}\\nTriggerAt: {{.trigger_at | datetime}}\"\n },{\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n }\n ]\n}\n}" - }, - "sub_type": "feishu", - "enabled": false -} -POST $[[INDEX_PREFIX]]channel/doc/cj8e9s53q95gsdbb054g -{ - "id": "cj8e9s53q95gsdbb054g", - "created": "2023-08-07T20:34:56.334695598+08:00", - "updated": "2023-08-08T21:34:50.261294305+08:00", - "name": "[Recovery] Feishu Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI Platform Alerting ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"green\"\n },\n \"elements\": [\n {\n \"tag\": \"markdown\",\n \"content\": \"šŸŒˆ **{{.title}}**\"\n },\n {\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n }\n ]\n }\n}" - }, - "sub_type": "feishu", - "enabled": false -} -POST $[[INDEX_PREFIX]]channel/doc/cj8ctat3q95l9ebbntlg - { - "id": "cj8ctat3q95l9ebbntlg", - "created": "2023-08-07T18:59:55.28732241+08:00", - "updated": "2023-08-08T19:46:30.557046793+08:00", - "name": "[Recovery] DingTalk Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.com/img/email/recovery-header.png)\\n\\nšŸŒˆ **{{.title}}**\\n\\n{{.message}}\\n\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n }\n}\n" - }, - "sub_type": "dingtalk", - "enabled": false -} -POST $[[INDEX_PREFIX]]channel/doc/cgnb2kt3q95nmusjl64g -{ - "id": "cgnb2kt3q95nmusjl64g", - "created": "2023-04-06T11:47:31.161587662Z", - "updated": "2023-08-08T22:19:06.712911427+08:00", - "name": "[Alerting] Wechat Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.WECOM_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"**[ INFINI Platform Alerting ]**\\nšŸ”„ Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing !\\n**{{.title}}**\\nPriority: {{.priority}}\\n\\nEventID: {{.event_id}}\\n\\nTarget: {{.resource_name}}-{{.objects}}\\n\\nTriggerAt: {{.trigger_at | datetime}}\\n{{.message}}\"\n }\n}" - }, - "sub_type": "wechat", - "enabled": false -} -POST $[[INDEX_PREFIX]]channel/doc/cj8e9gt3q95gsdbb0170 -{ - "id": "cj8e9gt3q95gsdbb0170", - "created": "2023-08-07T20:34:11.998953512+08:00", - "updated": "2023-08-08T19:47:08.270014715+08:00", - "name": "[Recovery] Wechat Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.WECOM_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"**[ INFINI Platform Alerting ]**\\nšŸŒˆ **{{.title}}**\\n\\n{{.message}}\\n\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n }\n}\n" - }, - "sub_type": "wechat", - "enabled": false -} POST $[[INDEX_PREFIX]]channel/doc/cj8atf53q95lhahebg8g { "id": "cj8atf53q95lhahebg8g", "created": "2023-08-07T16:43:40.062389175+08:00", - "updated": "2023-08-08T19:50:15.803258835+08:00", + "updated": "2023-08-10T12:04:42.842628127+08:00", "name": "[Recovery] Email Notification", "type": "email", "sub_type": "email", @@ -762,20 +690,93 @@ POST $[[INDEX_PREFIX]]channel/doc/cj8atf53q95lhahebg8g "to": [], "cc": [] }, - "subject": "[INFINI Platform Alerting] šŸŒˆ {{.title}}", - "body": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n
\n \"email-header\"\n
\n
\n \n \n \n \n \n \n
\n
\n
\n šŸŒˆ {{.title}}\n

\n {{.message | md_to_html}}\n
\n

\n \n \n View Detail\n \n

\n \n \n \n

\n \n

\n \"INFINI\n
\n
\n \n
\n

\n \n \n
\n
\n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n
\n
\n \n \n \n
\n
\n \n
\n \n \n
\n
\n \n ", + "subject": "[INFINI Platform Alerting] {{.title}}", + "body": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n
\n \"email-header\"\n
\n
\n \n \n \n \n \n \n
\n
\n
\n {{.title}}\n

\n {{.message | md_to_html}}\n
\n

\n \n \n View Detail\n \n

\n \n \n \n

\n \n

\n \"INFINI\n
\n
\n \n
\n

\n \n \n
\n
\n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n
\n
\n \n \n \n
\n
\n \n
\n \n \n
\n
\n \n ", "content_type": "text/html" }, "enabled": false } +POST $[[INDEX_PREFIX]]channel/doc/ch1os6t3q95lk6lepkq0 +{ + "id": "ch1os6t3q95lk6lepkq0", + "created": "2023-04-22T07:34:51.848540351Z", + "updated": "2023-08-10T17:18:38.592432088+08:00", + "name": "[Alerting] Feishu Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI Platform Alerting ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"{{if eq .priority \"critical\"}}red{{else if eq .priority \"high\"}}orange{{else if eq .priority \"medium\"}}yellow{{else if eq .priority \"low\"}}grey{{else}}blue{{end}}\"\n },\n \"elements\": [{\n \"tag\": \"markdown\",\n \"content\": \"šŸ”„ Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n **{{.title}}**\\nPriority: {{.priority}}\\nEventID: {{.event_id}}\\nTarget: {{.resource_name}}-{{.objects}}\\nTriggerAt: {{.trigger_at | datetime}}\"\n },{\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n }\n ]\n}\n}" + }, + "sub_type": "feishu", + "enabled": false + } +POST $[[INDEX_PREFIX]]channel/doc/cj8e9s53q95gsdbb054g +{ + "id": "cj8e9s53q95gsdbb054g", + "created": "2023-08-07T20:34:56.334695598+08:00", + "updated": "2023-08-10T17:18:36.035896482+08:00", + "name": "[Recovery] Feishu Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI Platform Alerting ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"green\"\n },\n \"elements\": [\n {\n \"tag\": \"markdown\",\n \"content\": \"šŸŒˆ **{{.title}}**\"\n },\n {\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n },\n {\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"[View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n }\n ]\n }\n}" + }, + "sub_type": "feishu", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cj865st3q95rega919ig +{ + "id": "cj865st3q95rega919ig", + "created": "2023-08-07T11:20:19.223545026+08:00", + "updated": "2023-08-10T17:18:41.92016786+08:00", + "name": "[Alerting] Discord Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DISCORD_WEBHOOK_ENDPOINT}}", + "body": "{\"content\": \"**[ INFINI Platform Alerting ]**\\nšŸ”„ Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n**{{.title}}**\\n\\nPriority: {{.priority}}\\nEventID: {{.event_id}}\\nTarget: {{.resource_name}}-{{.objects}}\\nTriggerAt: {{.trigger_at | datetime}}\\n{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"}" + }, + "sub_type": "discord", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cj86l0l3q95rrpfea6ug +{ + "id": "cj86l0l3q95rrpfea6ug", + "created": "2023-08-07T11:52:34.192522006+08:00", + "updated": "2023-08-10T17:18:44.422687739+08:00", + "name": "[Recovery] Discord Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DISCORD_WEBHOOK_ENDPOINT}}", + "body": "{\n \"content\": \"**[ INFINI Platform Alerting ]**\\nšŸŒˆ **{{.title}}**\\n\\n{{.message | str_replace \"\\n\" \"\\\\n\" }}\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n}" + }, + "sub_type": "discord", + "enabled": false +} #alerting -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6 +#The `id` value is consistent with the `_id` value +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0 { - "id": "builtin-calgapp7h710dpnpbeb6", - "created": "2022-06-16T10:26:47.360988761Z", - "updated": "2023-08-09T09:44:58.584645596+08:00", - "name": "Shard Storage >= 55G", + "id": "builtin-cal8n7p7h710dpnoaps0", + "created": "2022-06-16T01:47:11.326727124Z", + "updated": "2023-08-09T22:39:43.98598502+08:00", + "name": "Cluster Health Change to Red", "enabled": true, "resource": { "resource_id": "$[[RESOURCE_ID]]", @@ -786,10 +787,21 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6 ], "filter": {}, "raw_filter": { - "range": { - "payload.elasticsearch.index_stats.shard_info.store_in_bytes": { - "gte": 59055800320 - } + "bool": { + "must": [ + { + "match": { + "payload.elasticsearch.cluster_health.status": "red" + } + }, + { + "term": { + "metadata.name": { + "value": "cluster_health" + } + } + } + ] } }, "time_field": "timestamp", @@ -803,25 +815,21 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6 { "field": "metadata.labels.cluster_id", "limit": 5 - }, - { - "field": "metadata.labels.index_name", - "limit": 500 } ], "formula": "a", "items": [ { "name": "a", - "field": "payload.elasticsearch.index_stats.shard_info.store_in_bytes", - "statistic": "max" + "field": "payload.elasticsearch.cluster_health.status", + "statistic": "count" } ], - "format_type": "bytes", + "format_type": "num", "bucket_label": { "enabled": false }, - "expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)" + "expression": "count(payload.elasticsearch.cluster_health.status)" }, "conditions": { "operator": "any", @@ -830,40 +838,39 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6 "minimum_period_match": 1, "operator": "gte", "values": [ - "59055800320" + "1" ], - "priority": "high" + "priority": "critical" } ] }, "notification_config": { "enabled": true, - "title": "Shard Storage >55GB in ({{len .results}} indices in total)", - "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}), Max Shard Storage: {{.result_value | format_bytes 2}}\n{{end}}", + "title": "Health of Clusters ({{len .results}} clusters in total) Changed to Red", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nCluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) is Red now\n{{end}}", "normal": [ { "id": "cgnb2nt3q95nmusjl65g", "created": "2023-04-06T11:47:43.104108279Z", - "updated": "2023-08-07T14:02:53.734855705+08:00", + "updated": "2023-08-07T15:02:17.165625799+08:00", "name": "[Alerting] Slack Notification", "type": "webhook", "webhook": { "header_params": { - "Content-type": "application/json" + "Content-Type": "application/json" }, "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Max shard storage: {{.result_value | format_bytes 2}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Cluster:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}> is Red now\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" }, "sub_type": "slack", "enabled": true }, { - "id": "cj865st3q95rega919ig", + "id": "cgiospt3q95q49k3u00g", "enabled": true }, { - "id": "cgiospt3q95q49k3u00g", + "id": "cj865st3q95rega919ig", "enabled": true }, { @@ -887,8 +894,160 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6 }, "recovery_notification_config": { "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "- EventID: {{.event_id}}\n- Target: {{.resource_name}}-{{.objects}}\n- TriggerAt: {{.trigger_at}}\n- ResolveAt: {{.timestamp | datetime}}\n- Duration: {{.duration}}", + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calavvp7h710dpnp32r3 +{ + "id": "builtin-calavvp7h710dpnp32r3", + "created": "2022-06-16T04:22:23.001354546Z", + "updated": "2023-08-09T22:20:17.864619426+08:00", + "name": "Index Health Change to Red", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_index" + ], + "filter": {}, + "raw_filter": { + "match": { + "metadata.labels.health_status": "red" + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.cluster_id", + "limit": 50 + }, + { + "field": "metadata.index_name", + "limit": 1000 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "metadata.index_name", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(metadata.index_name)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Health of Indices ({{len .results}} indices in total) Changed to Red", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nIndex: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D) is Red now\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T15:17:26.18861218+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{ lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0)}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}> is Red now\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", "normal": [ { "id": "cj8bq8d3q95ogankugqg", @@ -925,11 +1084,163 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6 "id": "$[[USER_ID]]" } } +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp20n2anisjmu4gehc5 +{ + "id": "builtin-cbp20n2anisjmu4gehc5", + "created": "2022-08-09T08:52:44.63345561Z", + "updated": "2023-08-09T22:11:45.679048697+08:00", + "name": "Elasticsearch node left cluster", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_node" + ], + "filter": {}, + "raw_filter": { + "match_phrase": { + "metadata.labels.status": "unavailable" + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.cluster_id", + "limit": 5 + }, + { + "field": "metadata.node_id", + "limit": 50 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "metadata.labels.status", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(metadata.labels.status)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "critical" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Elasticsearch node left cluster", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nNode: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Left: {{.result_value}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T10:42:17.686776304+08:00", + "name": "Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Left: {{.result_value}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cb34sfl6psfiqtovhpt4 { "id": "builtin-cb34sfl6psfiqtovhpt4", "created": "2022-07-07T03:08:46.297166036Z", - "updated": "2023-08-09T09:45:34.123901475+08:00", + "updated": "2023-08-09T22:38:41.764325087+08:00", "name": "Too Many Deleted Documents", "enabled": true, "resource": { @@ -1015,7 +1326,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cb34sfl6psfiqtovhpt4 "notification_config": { "enabled": true, "title": "Too Many Deleted Documents (>30%)", - "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), Deleted: {{.result_value | to_fixed 2}}%\n{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nIndex: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Deleted: {{.result_value | to_fixed 2}}%\n{{end}}", "normal": [ { "id": "cgnb2nt3q95nmusjl65g", @@ -1026,7 +1337,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cb34sfl6psfiqtovhpt4 "Content-type": "application/json" }, "method": "POST", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Deleted ratio: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{ lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0)}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Deleted ratio: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" }, "sub_type": "", "enabled": true @@ -1060,314 +1371,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cb34sfl6psfiqtovhpt4 }, "recovery_notification_config": { "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", - "normal": [ - { - "id": "cj8bq8d3q95ogankugqg", - "enabled": true - }, - { - "id": "cj8ctat3q95l9ebbntlg", - "enabled": true - }, - { - "id": "cj8atf53q95lhahebg8g", - "enabled": true - }, - { - "id": "cj8e9s53q95gsdbb054g", - "enabled": true - }, - { - "id": "cj8e9gt3q95gsdbb0170", - "enabled": true - }, - { - "id": "cj86l0l3q95rrpfea6ug", - "enabled": true - } - ], - "event_enabled": true - }, - "schedule": { - "interval": "1m" - }, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - } -} - -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp20n2anisjmu4gehc5 -{ - "id": "builtin-cbp20n2anisjmu4gehc5", - "created": "2022-08-09T08:52:44.63345561Z", - "updated": "2023-08-09T09:43:37.945659792+08:00", - "name": "Elasticsearch node left cluster", - "enabled": true, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - ".infini_node" - ], - "filter": {}, - "raw_filter": { - "match_phrase": { - "metadata.labels.status": "unavailable" - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.cluster_id", - "limit": 5 - }, - { - "field": "metadata.node_id", - "limit": 50 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "metadata.labels.status", - "statistic": "count" - } - ], - "format_type": "num", - "bucket_label": { - "enabled": false - }, - "expression": "count(metadata.labels.status)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "1" - ], - "priority": "critical" - } - ] - }, - "notification_config": { - "enabled": true, - "title": "Elasticsearch node left cluster", - "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), Left: {{.result_value}}\n{{end}}", - "normal": [ - { - "id": "cgnb2nt3q95nmusjl65g", - "created": "2023-04-06T11:47:43.104108279Z", - "updated": "2023-08-07T10:42:17.686776304+08:00", - "name": "Slack Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-type": "application/json" - }, - "method": "POST", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Left: {{.result_value}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" - }, - "sub_type": "slack", - "enabled": true - }, - { - "id": "cgnb2r53q95nmusjl6vg", - "enabled": true - }, - { - "id": "ch1os6t3q95lk6lepkq0", - "enabled": true - }, - { - "id": "cgnb2kt3q95nmusjl64g", - "enabled": true - }, - { - "id": "cj865st3q95rega919ig", - "enabled": true - }, - { - "id": "cgiospt3q95q49k3u00g", - "enabled": true - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "recovery_notification_config": { - "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", - "normal": [ - { - "id": "cj8bq8d3q95ogankugqg", - "enabled": true - }, - { - "id": "cj8ctat3q95l9ebbntlg", - "enabled": true - }, - { - "id": "cj8e9s53q95gsdbb054g", - "enabled": true - }, - { - "id": "cj8e9gt3q95gsdbb0170", - "enabled": true - }, - { - "id": "cj86l0l3q95rrpfea6ug", - "enabled": true - }, - { - "id": "cj8atf53q95lhahebg8g", - "enabled": true - } - ], - "event_enabled": true - }, - "schedule": { - "interval": "1m" - }, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - } -} -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calavvp7h710dpnp32r3 -{ - "id": "builtin-calavvp7h710dpnp32r3", - "created": "2022-06-16T04:22:23.001354546Z", - "updated": "2023-08-09T09:43:58.551403706+08:00", - "name": "Index Health Change to Red", - "enabled": true, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - ".infini_index" - ], - "filter": {}, - "raw_filter": { - "match": { - "metadata.labels.health_status": "red" - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.cluster_id", - "limit": 50 - }, - { - "field": "metadata.index_name", - "limit": 1000 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "metadata.index_name", - "statistic": "count" - } - ], - "format_type": "num", - "bucket_label": { - "enabled": false - }, - "expression": "count(metadata.index_name)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "1" - ], - "priority": "high" - } - ] - }, - "notification_config": { - "enabled": true, - "title": "Health of Indices ({{len .results}} indices in total) Changed to Red", - "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}) is Red now\n{{end}}", - "normal": [ - { - "id": "cgnb2nt3q95nmusjl65g", - "created": "2023-04-06T11:47:43.104108279Z", - "updated": "2023-08-07T15:17:26.18861218+08:00", - "name": "[Alerting] Slack Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}> is Red now\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" - }, - "sub_type": "slack", - "enabled": true - }, - { - "id": "cgiospt3q95q49k3u00g", - "enabled": true - }, - { - "id": "cj865st3q95rega919ig", - "enabled": true - }, - { - "id": "cgnb2r53q95nmusjl6vg", - "enabled": true - }, - { - "id": "ch1os6t3q95lk6lepkq0", - "enabled": true - }, - { - "id": "cgnb2kt3q95nmusjl64g", - "enabled": true - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "recovery_notification_config": { - "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", "normal": [ { "id": "cj8bq8d3q95ogankugqg", @@ -1408,7 +1413,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7 { "id": "builtin-cbp2e4ianisjmu4giqs7", "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2023-08-09T09:44:31.495696286+08:00", + "updated": "2023-08-09T22:39:15.339913317+08:00", "name": "Search latency is great than 500ms", "enabled": true, "resource": { @@ -1508,7 +1513,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7 "notification_config": { "enabled": true, "title": "Search latency is great than 500ms", - "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}), Latency: {{.result_value | to_fixed 2}}ms\n{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nIndex: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Latency: {{.result_value | to_fixed 2}}ms\n{{end}}", "normal": [ { "id": "cgnb2nt3q95nmusjl65g", @@ -1521,8 +1526,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7 "Content-type": "application/json" }, "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "\n{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Search latency: {{.result_value | to_fixed 2}}ms\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + "body": "\n{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{ lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0)}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Search latency: {{.result_value | to_fixed 2}}ms\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" }, "sub_type": "slack", "enabled": true @@ -1556,8 +1560,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7 }, "recovery_notification_config": { "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", "normal": [ { "id": "cj8bq8d3q95ogankugqg", @@ -1598,7 +1602,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8 { "id": "builtin-calaqnh7h710dpnp2bm8", "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2023-08-09T09:46:34.428920151+08:00", + "updated": "2023-08-09T22:38:55.677122718+08:00", "name": "JVM utilization is Too High", "enabled": true, "resource": { @@ -1692,7 +1696,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8 "notification_config": { "enabled": true, "title": "JVM Usage of Nodes ({{len .results}} nodes in total) >= {{.first_threshold}}%", - "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), JVM Usage: {{.result_value | to_fixed 2}}%\n{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nNode: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), JVM Usage: {{.result_value | to_fixed 2}}%\n{{end}}", "normal": [ { "id": "cgnb2nt3q95nmusjl65g", @@ -1705,7 +1709,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8 "Content-type": "application/json" }, "method": "POST", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, JVM Usage: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, JVM Usage: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" }, "sub_type": "slack", "enabled": true @@ -1739,8 +1743,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8 }, "recovery_notification_config": { "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", "normal": [ { "id": "cj8bq8d3q95ogankugqg", @@ -1781,7 +1785,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2 { "id": "builtin-calakp97h710dpnp1fa2", "created": "2022-06-16T03:58:29.437447113Z", - "updated": "2023-08-09T09:42:57.901272952+08:00", + "updated": "2023-08-09T22:33:25.692835454+08:00", "name": "CPU utilization is Too High", "enabled": true, "resource": { @@ -1850,7 +1854,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2 "minimum_period_match": 1, "operator": "gte", "values": [ - "85" + "80" ], "priority": "low" }, @@ -1875,7 +1879,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2 "notification_config": { "enabled": true, "title": "CPU Usage of Nodes ({{len .results}} nodes in total) >= {{.first_threshold}}%", - "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), CPU Usage: {{.result_value | to_fixed 2}}%\n{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nNode: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), CPU Usage: {{.result_value | to_fixed 2}}%\n{{end}}", "normal": [ { "id": "cgnb2nt3q95nmusjl65g", @@ -1888,8 +1892,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2 "Content-type": "application/json" }, "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, CPU Usage: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, CPU Usage: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" }, "sub_type": "slack", "enabled": true @@ -1923,8 +1926,162 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2 }, "recovery_notification_config": { "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6 +{ + "id": "builtin-calgapp7h710dpnpbeb6", + "created": "2022-06-16T10:26:47.360988761Z", + "updated": "2023-08-09T22:37:44.038127695+08:00", + "name": "Shard Storage >= 55G", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "range": { + "payload.elasticsearch.index_stats.shard_info.store_in_bytes": { + "gte": 59055800320 + } + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.index_name", + "limit": 500 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.index_stats.shard_info.store_in_bytes", + "statistic": "max" + } + ], + "format_type": "bytes", + "bucket_label": { + "enabled": false + }, + "expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "59055800320" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Shard Storage >55GB in ({{len .results}} indices in total)", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nIndex: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Max Shard Storage: {{.result_value | format_bytes 2}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T14:02:53.734855705+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{ lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0)}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Max shard storage: {{.result_value | format_bytes 2}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "- EventID: {{.event_id}}\n- Target: {{.resource_name}}-{{.objects}}\n- TriggerAt: {{.trigger_at | datetime}}\n- ResolveAt: {{.timestamp | datetime}}\n- Duration: {{.duration}}", "normal": [ { "id": "cj8bq8d3q95ogankugqg", @@ -1965,7 +2122,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnogps1 { "id": "builtin-cal8n7p7h710dpnogps1", "created": "2022-06-16T03:11:01.445958361Z", - "updated": "2023-08-09T09:43:16.31964237+08:00", + "updated": "2023-08-10T17:16:34.900352415+08:00", "name": "Disk utilization is Too High", "enabled": true, "resource": { @@ -2064,7 +2221,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnogps1 "notification_config": { "enabled": true, "title": "Disk Usage of Nodes ({{len .results}} nodes in total) >= {{.first_threshold}}%", - "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), Usage: {{.result_value | to_fixed 2}}% / Free: {{.relation_values.b | format_bytes 2}}\n{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nNode: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Usage: {{.result_value | to_fixed 2}}% / Free: {{.relation_values.b | format_bytes 2}}\n{{end}}", "normal": [ { "id": "cgnb2nt3q95nmusjl65g", @@ -2109,175 +2266,9 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnogps1 }, "recovery_notification_config": { "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", - "normal": [ - { - "id": "cj8bq8d3q95ogankugqg", - "enabled": true - }, - { - "id": "cj8ctat3q95l9ebbntlg", - "enabled": true - }, - { - "id": "cj8atf53q95lhahebg8g", - "enabled": true - }, - { - "id": "cj8e9s53q95gsdbb054g", - "enabled": true - }, - { - "id": "cj8e9gt3q95gsdbb0170", - "enabled": true - }, - { - "id": "cj86l0l3q95rrpfea6ug", - "enabled": true - } - ], - "event_enabled": true - }, - "schedule": { - "interval": "1m" - }, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - } -} -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0 -{ - "id": "builtin-cal8n7p7h710dpnoaps0", - "created": "2022-06-16T01:47:11.326727124Z", - "updated": "2023-08-09T09:50:05.833535441+08:00", - "name": "Cluster Health Change to Red", - "enabled": true, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - ".infini_metrics*" - ], - "filter": {}, - "raw_filter": { - "bool": { - "must": [ - { - "match": { - "payload.elasticsearch.cluster_health.status": "red" - } - }, - { - "term": { - "metadata.name": { - "value": "cluster_health" - } - } - } - ] - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.cluster_health.status", - "statistic": "count" - } - ], - "format_type": "num", - "bucket_label": { - "enabled": false - }, - "expression": "count(payload.elasticsearch.cluster_health.status)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "1" - ], - "priority": "critical" - } - ] - }, - "notification_config": { - "enabled": true, - "title": "Health of Clusters ({{len .results}} clusters in total) Changed to Red", - "message": "{{range .results}}\nCluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/) is Red now\n{{end}}", - "normal": [ - { - "id": "cgnb2nt3q95nmusjl65g", - "created": "2023-04-06T11:47:43.104108279Z", - "updated": "2023-08-07T15:02:17.165625799+08:00", - "name": "[Alerting] Slack Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Cluster:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}> is Red now\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" - }, - "sub_type": "slack", - "enabled": true - }, - { - "id": "cgiospt3q95q49k3u00g", - "enabled": true - }, - { - "id": "cj865st3q95rega919ig", - "enabled": true - }, - { - "id": "cgnb2r53q95nmusjl6vg", - "enabled": true - }, - { - "id": "ch1os6t3q95lk6lepkq0", - "enabled": true - }, - { - "id": "cgnb2kt3q95nmusjl64g", - "enabled": true - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "recovery_notification_config": { - "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", + "title": "šŸŒˆ [{{.rule_name}}] Resolved", "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", "normal": [ - { - "id": "cj8atf53q95lhahebg8g", - "enabled": true - }, { "id": "cj8bq8d3q95ogankugqg", "enabled": true @@ -2286,6 +2277,10 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0 "id": "cj8ctat3q95l9ebbntlg", "enabled": true }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, { "id": "cj8e9s53q95gsdbb054g", "enabled": true diff --git a/config/initialization_v6.tpl b/config/initialization_v6.tpl index 458c4506..7c3761a9 100644 --- a/config/initialization_v6.tpl +++ b/config/initialization_v6.tpl @@ -623,65 +623,29 @@ PUT $[[INDEX_PREFIX]]activities-00001 #alerting channel #The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]channel/doc/cj865st3q95rega919ig -{ - "id": "cj865st3q95rega919ig", - "created": "2023-08-07T11:20:19.223545026+08:00", - "updated": "2023-08-08T18:42:26.506499014+08:00", - "name": "[Alerting] Discord", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.DISCORD_WEBHOOK_ENDPOINT}}", - "body": "{\n \"content\": \"Hello Alerting\"\n}" - }, - "sub_type": "discord", - "enabled": false -} -POST $[[INDEX_PREFIX]]channel/doc/cj86l0l3q95rrpfea6ug -{ - "id": "cj86l0l3q95rrpfea6ug", - "created": "2023-08-07T11:52:34.192522006+08:00", - "updated": "2023-08-08T18:42:30.162079286+08:00", - "name": "[Recovery] Discord\t", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.DISCORD_WEBHOOK_ENDPOINT}}", - "body": "{\n\n}" - }, - "sub_type": "discord", - "enabled": false -} POST $[[INDEX_PREFIX]]channel/doc/cgnb2nt3q95nmusjl65g { - "id": "cgnb2nt3q95nmusjl65g", - "created": "2023-04-06T11:47:43.104108279Z", - "updated": "2023-08-08T22:19:08.601341574+08:00", - "name": "[Alerting] Slack Notification", - "type": "webhook", - "webhook": { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-09T22:39:50.494915568+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { "header_params": { "Content-type": "application/json" }, "method": "POST", "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing !*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.trigger_at | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Cluster:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" - }, - "sub_type": "slack", - "enabled": false + }, + "sub_type": "slack", + "enabled": false } POST $[[INDEX_PREFIX]]channel/doc/cj8bq8d3q95ogankugqg { "id": "cj8bq8d3q95ogankugqg", "created": "2023-08-07T17:45:05.534408059+08:00", - "updated": "2023-08-08T19:26:34.009668892+08:00", + "updated": "2023-08-09T22:39:56.489567891+08:00", "name": "[Recovery] Slack Notification", "type": "webhook", "webhook": { @@ -690,16 +654,34 @@ POST $[[INDEX_PREFIX]]channel/doc/cj8bq8d3q95ogankugqg }, "method": "POST", "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*:rainbow: Alert [{{.rule_name}}] Resolved*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*ResolveAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Target:* {{.resource_name}}-{{.objects}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.trigger_at | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Duration:* {{.duration}}\"\n }\n },\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n }\n ]\n}" + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*ResolveAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Target:* {{.resource_name}}-{{.objects}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.trigger_at | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Duration:* {{.duration}}\"\n }\n },\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n }\n ]\n}" }, "sub_type": "slack", "enabled": false } +POST $[[INDEX_PREFIX]]channel/doc/cgnb2kt3q95nmusjl64g +{ + "id": "cgnb2kt3q95nmusjl64g", + "created": "2023-04-06T11:47:31.161587662Z", + "updated": "2023-08-09T22:39:51.540172306+08:00", + "name": "[Alerting] Wechat Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.WECOM_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"**[ INFINI Platform Alerting ]**\\nšŸ”„ Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n**{{.title}}**\\nPriority: {{.priority}}\\n\\nEventID: {{.event_id}}\\n\\nTarget: {{.resource_name}}-{{.objects}}\\n\\nTriggerAt: {{.trigger_at | datetime}}\\n{{.message}}\"\n }\n}" + }, + "sub_type": "wechat", + "enabled": false +} POST $[[INDEX_PREFIX]]channel/doc/cgiospt3q95q49k3u00g { "id": "cgiospt3q95q49k3u00g", "created": "2023-03-30T13:28:07.531263747Z", - "updated": "2023-08-08T22:19:07.545051029+08:00", + "updated": "2023-08-09T22:39:52.356059486+08:00", "name": "[Alerting] DingTalk Notification", "type": "webhook", "webhook": { @@ -708,16 +690,52 @@ POST $[[INDEX_PREFIX]]channel/doc/cgiospt3q95q49k3u00g }, "method": "POST", "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.com/img/email/alert-header.png)\\n\\nšŸ”„ **{{.title}}**\\n\\nIncident [{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing !\\n\\nPriority: {{.priority}}\\n\\nEventID: {{.event_id}}\\n\\nTarget: {{.resource_name}}-{{.objects}}\\n\\nTriggerAt: {{.trigger_at | datetime}}\\n\\n---\\n\\n{{.message}}\"\n }\n}" + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.com/img/email/alert-header.png)\\n\\nšŸ”„ Incident [{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n\\n**{{.title}}**\\n\\nPriority: {{.priority}}\\n\\nEventID: {{.event_id}}\\n\\nTarget: {{.resource_name}}-{{.objects}}\\n\\nTriggerAt: {{.trigger_at | datetime}}\\n\\n---\\n\\n{{.message}}\"\n }\n}" }, "sub_type": "dingtalk", "enabled": false } +POST $[[INDEX_PREFIX]]channel/doc/cj8ctat3q95l9ebbntlg +{ + "id": "cj8ctat3q95l9ebbntlg", + "created": "2023-08-07T18:59:55.28732241+08:00", + "updated": "2023-08-09T22:39:58.967970184+08:00", + "name": "[Recovery] DingTalk Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.com/img/email/recovery-header.png)\\n\\n**{{.title}}**\\n\\n{{.message}}\\n\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n }\n}\n" + }, + "sub_type": "dingtalk", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cj8e9gt3q95gsdbb0170 +{ + "id": "cj8e9gt3q95gsdbb0170", + "created": "2023-08-07T20:34:11.998953512+08:00", + "updated": "2023-08-09T22:40:04.665871275+08:00", + "name": "[Recovery] Wechat Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.WECOM_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"**[ INFINI Platform Alerting ]**\\n**{{.title}}**\\n\\n{{.message}}\\n\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n }\n}\n" + }, + "sub_type": "wechat", + "enabled": false +} POST $[[INDEX_PREFIX]]channel/doc/cgnb2r53q95nmusjl6vg { "id": "cgnb2r53q95nmusjl6vg", "created": "2023-04-06T11:47:56.652637309Z", - "updated": "2023-08-08T19:49:20.312590885+08:00", + "updated": "2023-08-10T12:04:08.046781556+08:00", "name": "[Alerting] Email Notification", "type": "email", "sub_type": "email", @@ -729,105 +747,15 @@ POST $[[INDEX_PREFIX]]channel/doc/cgnb2r53q95nmusjl6vg }, "subject": "[INFINI Platform Alerting] šŸ”„ {{.title}}", "body": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n
\n \"email-header\"\n
\n
\n \n \n \n \n \n \n
\n
\n
\n {{.title}}\n

\n \n

Priority: {{.priority}}

\n

EventID: {{.event_id}}

\n

Target: {{.resource_name}}-{{.objects}}

\n

TriggerAt: {{.trigger_at | datetime}}

\n {{.message | md_to_html}}\n
\n

\n \n \n View Detail\n \n

\n \n \n \n

\n \n

\n \"INFINI\n
\n
\n \n
\n

\n \n \n
\n
\n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n
\n
\n \n \n \n
\n
\n \n
\n \n \n
\n
\n \n ", - "content_type": "text/html" + "content_type": "text/html" }, "enabled": false } -POST $[[INDEX_PREFIX]]channel/doc/ch1os6t3q95lk6lepkq0 -{ - "id": "ch1os6t3q95lk6lepkq0", - "created": "2023-04-22T07:34:51.848540351Z", - "updated": "2023-08-09T09:29:26.412223281+08:00", - "name": "[Alerting] Feishu Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI Platform Alerting ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"{{if eq .priority \"critical\"}}red{{else if eq .priority \"high\"}}orange{{else if eq .priority \"medium\"}}yellow{{else if eq .priority \"low\"}}grey{{else}}blue{{end}}\"\n },\n \"elements\": [{\n \"tag\": \"markdown\",\n \"content\": \"šŸ”„ Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing ! \\n **{{.title}}**\\nPriority: {{.priority}}\\nEventID: {{.event_id}}\\nTarget: {{.resource_name}}-{{.objects}}\\nTriggerAt: {{.trigger_at | datetime}}\"\n },{\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n }\n ]\n}\n}" - }, - "sub_type": "feishu", - "enabled": false -} -POST $[[INDEX_PREFIX]]channel/doc/cj8e9s53q95gsdbb054g -{ - "id": "cj8e9s53q95gsdbb054g", - "created": "2023-08-07T20:34:56.334695598+08:00", - "updated": "2023-08-08T21:34:50.261294305+08:00", - "name": "[Recovery] Feishu Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI Platform Alerting ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"green\"\n },\n \"elements\": [\n {\n \"tag\": \"markdown\",\n \"content\": \"šŸŒˆ **{{.title}}**\"\n },\n {\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n }\n ]\n }\n}" - }, - "sub_type": "feishu", - "enabled": false -} -POST $[[INDEX_PREFIX]]channel/doc/cj8ctat3q95l9ebbntlg - { - "id": "cj8ctat3q95l9ebbntlg", - "created": "2023-08-07T18:59:55.28732241+08:00", - "updated": "2023-08-08T19:46:30.557046793+08:00", - "name": "[Recovery] DingTalk Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.com/img/email/recovery-header.png)\\n\\nšŸŒˆ **{{.title}}**\\n\\n{{.message}}\\n\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n }\n}\n" - }, - "sub_type": "dingtalk", - "enabled": false -} -POST $[[INDEX_PREFIX]]channel/doc/cgnb2kt3q95nmusjl64g -{ - "id": "cgnb2kt3q95nmusjl64g", - "created": "2023-04-06T11:47:31.161587662Z", - "updated": "2023-08-08T22:19:06.712911427+08:00", - "name": "[Alerting] Wechat Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.WECOM_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"**[ INFINI Platform Alerting ]**\\nšŸ”„ Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing !\\n**{{.title}}**\\nPriority: {{.priority}}\\n\\nEventID: {{.event_id}}\\n\\nTarget: {{.resource_name}}-{{.objects}}\\n\\nTriggerAt: {{.trigger_at | datetime}}\\n{{.message}}\"\n }\n}" - }, - "sub_type": "wechat", - "enabled": false -} -POST $[[INDEX_PREFIX]]channel/doc/cj8e9gt3q95gsdbb0170 -{ - "id": "cj8e9gt3q95gsdbb0170", - "created": "2023-08-07T20:34:11.998953512+08:00", - "updated": "2023-08-08T19:47:08.270014715+08:00", - "name": "[Recovery] Wechat Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.WECOM_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"**[ INFINI Platform Alerting ]**\\nšŸŒˆ **{{.title}}**\\n\\n{{.message}}\\n\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n }\n}\n" - }, - "sub_type": "wechat", - "enabled": false -} POST $[[INDEX_PREFIX]]channel/doc/cj8atf53q95lhahebg8g { "id": "cj8atf53q95lhahebg8g", "created": "2023-08-07T16:43:40.062389175+08:00", - "updated": "2023-08-08T19:50:15.803258835+08:00", + "updated": "2023-08-10T12:04:42.842628127+08:00", "name": "[Recovery] Email Notification", "type": "email", "sub_type": "email", @@ -837,20 +765,93 @@ POST $[[INDEX_PREFIX]]channel/doc/cj8atf53q95lhahebg8g "to": [], "cc": [] }, - "subject": "[INFINI Platform Alerting] šŸŒˆ {{.title}}", - "body": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n
\n \"email-header\"\n
\n
\n \n \n \n \n \n \n
\n
\n
\n šŸŒˆ {{.title}}\n

\n {{.message | md_to_html}}\n
\n

\n \n \n View Detail\n \n

\n \n \n \n

\n \n

\n \"INFINI\n
\n
\n \n
\n

\n \n \n
\n
\n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n
\n
\n \n \n \n
\n
\n \n
\n \n \n
\n
\n \n ", + "subject": "[INFINI Platform Alerting] {{.title}}", + "body": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n
\n \"email-header\"\n
\n
\n \n \n \n \n \n \n
\n
\n
\n {{.title}}\n

\n {{.message | md_to_html}}\n
\n

\n \n \n View Detail\n \n

\n \n \n \n

\n \n

\n \"INFINI\n
\n
\n \n
\n

\n \n \n
\n
\n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n
\n
\n \n \n \n
\n
\n \n
\n \n \n
\n
\n \n ", "content_type": "text/html" }, "enabled": false } +POST $[[INDEX_PREFIX]]channel/doc/ch1os6t3q95lk6lepkq0 +{ + "id": "ch1os6t3q95lk6lepkq0", + "created": "2023-04-22T07:34:51.848540351Z", + "updated": "2023-08-10T17:18:38.592432088+08:00", + "name": "[Alerting] Feishu Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI Platform Alerting ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"{{if eq .priority \"critical\"}}red{{else if eq .priority \"high\"}}orange{{else if eq .priority \"medium\"}}yellow{{else if eq .priority \"low\"}}grey{{else}}blue{{end}}\"\n },\n \"elements\": [{\n \"tag\": \"markdown\",\n \"content\": \"šŸ”„ Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n **{{.title}}**\\nPriority: {{.priority}}\\nEventID: {{.event_id}}\\nTarget: {{.resource_name}}-{{.objects}}\\nTriggerAt: {{.trigger_at | datetime}}\"\n },{\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n }\n ]\n}\n}" + }, + "sub_type": "feishu", + "enabled": false + } +POST $[[INDEX_PREFIX]]channel/doc/cj8e9s53q95gsdbb054g +{ + "id": "cj8e9s53q95gsdbb054g", + "created": "2023-08-07T20:34:56.334695598+08:00", + "updated": "2023-08-10T17:18:36.035896482+08:00", + "name": "[Recovery] Feishu Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI Platform Alerting ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"green\"\n },\n \"elements\": [\n {\n \"tag\": \"markdown\",\n \"content\": \"šŸŒˆ **{{.title}}**\"\n },\n {\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n },\n {\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"[View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n }\n ]\n }\n}" + }, + "sub_type": "feishu", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cj865st3q95rega919ig +{ + "id": "cj865st3q95rega919ig", + "created": "2023-08-07T11:20:19.223545026+08:00", + "updated": "2023-08-10T17:18:41.92016786+08:00", + "name": "[Alerting] Discord Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DISCORD_WEBHOOK_ENDPOINT}}", + "body": "{\"content\": \"**[ INFINI Platform Alerting ]**\\nšŸ”„ Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n**{{.title}}**\\n\\nPriority: {{.priority}}\\nEventID: {{.event_id}}\\nTarget: {{.resource_name}}-{{.objects}}\\nTriggerAt: {{.trigger_at | datetime}}\\n{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"}" + }, + "sub_type": "discord", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cj86l0l3q95rrpfea6ug +{ + "id": "cj86l0l3q95rrpfea6ug", + "created": "2023-08-07T11:52:34.192522006+08:00", + "updated": "2023-08-10T17:18:44.422687739+08:00", + "name": "[Recovery] Discord Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DISCORD_WEBHOOK_ENDPOINT}}", + "body": "{\n \"content\": \"**[ INFINI Platform Alerting ]**\\nšŸŒˆ **{{.title}}**\\n\\n{{.message | str_replace \"\\n\" \"\\\\n\" }}\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n}" + }, + "sub_type": "discord", + "enabled": false +} #alerting -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6 +#The `id` value is consistent with the `_id` value +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0 { - "id": "builtin-calgapp7h710dpnpbeb6", - "created": "2022-06-16T10:26:47.360988761Z", - "updated": "2023-08-09T09:44:58.584645596+08:00", - "name": "Shard Storage >= 55G", + "id": "builtin-cal8n7p7h710dpnoaps0", + "created": "2022-06-16T01:47:11.326727124Z", + "updated": "2023-08-09T22:39:43.98598502+08:00", + "name": "Cluster Health Change to Red", "enabled": true, "resource": { "resource_id": "$[[RESOURCE_ID]]", @@ -861,10 +862,21 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6 ], "filter": {}, "raw_filter": { - "range": { - "payload.elasticsearch.index_stats.shard_info.store_in_bytes": { - "gte": 59055800320 - } + "bool": { + "must": [ + { + "match": { + "payload.elasticsearch.cluster_health.status": "red" + } + }, + { + "term": { + "metadata.name": { + "value": "cluster_health" + } + } + } + ] } }, "time_field": "timestamp", @@ -878,25 +890,21 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6 { "field": "metadata.labels.cluster_id", "limit": 5 - }, - { - "field": "metadata.labels.index_name", - "limit": 500 } ], "formula": "a", "items": [ { "name": "a", - "field": "payload.elasticsearch.index_stats.shard_info.store_in_bytes", - "statistic": "max" + "field": "payload.elasticsearch.cluster_health.status", + "statistic": "count" } ], - "format_type": "bytes", + "format_type": "num", "bucket_label": { "enabled": false }, - "expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)" + "expression": "count(payload.elasticsearch.cluster_health.status)" }, "conditions": { "operator": "any", @@ -905,40 +913,39 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6 "minimum_period_match": 1, "operator": "gte", "values": [ - "59055800320" + "1" ], - "priority": "high" + "priority": "critical" } ] }, "notification_config": { "enabled": true, - "title": "Shard Storage >55GB in ({{len .results}} indices in total)", - "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}), Max Shard Storage: {{.result_value | format_bytes 2}}\n{{end}}", + "title": "Health of Clusters ({{len .results}} clusters in total) Changed to Red", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nCluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) is Red now\n{{end}}", "normal": [ { "id": "cgnb2nt3q95nmusjl65g", "created": "2023-04-06T11:47:43.104108279Z", - "updated": "2023-08-07T14:02:53.734855705+08:00", + "updated": "2023-08-07T15:02:17.165625799+08:00", "name": "[Alerting] Slack Notification", "type": "webhook", "webhook": { "header_params": { - "Content-type": "application/json" + "Content-Type": "application/json" }, "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Max shard storage: {{.result_value | format_bytes 2}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Cluster:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}> is Red now\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" }, "sub_type": "slack", "enabled": true }, { - "id": "cj865st3q95rega919ig", + "id": "cgiospt3q95q49k3u00g", "enabled": true }, { - "id": "cgiospt3q95q49k3u00g", + "id": "cj865st3q95rega919ig", "enabled": true }, { @@ -962,8 +969,160 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6 }, "recovery_notification_config": { "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "- EventID: {{.event_id}}\n- Target: {{.resource_name}}-{{.objects}}\n- TriggerAt: {{.trigger_at}}\n- ResolveAt: {{.timestamp | datetime}}\n- Duration: {{.duration}}", + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calavvp7h710dpnp32r3 +{ + "id": "builtin-calavvp7h710dpnp32r3", + "created": "2022-06-16T04:22:23.001354546Z", + "updated": "2023-08-09T22:20:17.864619426+08:00", + "name": "Index Health Change to Red", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_index" + ], + "filter": {}, + "raw_filter": { + "match": { + "metadata.labels.health_status": "red" + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.cluster_id", + "limit": 50 + }, + { + "field": "metadata.index_name", + "limit": 1000 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "metadata.index_name", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(metadata.index_name)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Health of Indices ({{len .results}} indices in total) Changed to Red", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nIndex: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D) is Red now\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T15:17:26.18861218+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{ lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0)}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}> is Red now\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", "normal": [ { "id": "cj8bq8d3q95ogankugqg", @@ -1000,11 +1159,163 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6 "id": "$[[USER_ID]]" } } +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp20n2anisjmu4gehc5 +{ + "id": "builtin-cbp20n2anisjmu4gehc5", + "created": "2022-08-09T08:52:44.63345561Z", + "updated": "2023-08-09T22:11:45.679048697+08:00", + "name": "Elasticsearch node left cluster", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_node" + ], + "filter": {}, + "raw_filter": { + "match_phrase": { + "metadata.labels.status": "unavailable" + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.cluster_id", + "limit": 5 + }, + { + "field": "metadata.node_id", + "limit": 50 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "metadata.labels.status", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(metadata.labels.status)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "critical" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Elasticsearch node left cluster", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nNode: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Left: {{.result_value}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T10:42:17.686776304+08:00", + "name": "Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Left: {{.result_value}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cb34sfl6psfiqtovhpt4 { "id": "builtin-cb34sfl6psfiqtovhpt4", "created": "2022-07-07T03:08:46.297166036Z", - "updated": "2023-08-09T09:45:34.123901475+08:00", + "updated": "2023-08-09T22:38:41.764325087+08:00", "name": "Too Many Deleted Documents", "enabled": true, "resource": { @@ -1090,7 +1401,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cb34sfl6psfiqtovhpt4 "notification_config": { "enabled": true, "title": "Too Many Deleted Documents (>30%)", - "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), Deleted: {{.result_value | to_fixed 2}}%\n{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nIndex: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Deleted: {{.result_value | to_fixed 2}}%\n{{end}}", "normal": [ { "id": "cgnb2nt3q95nmusjl65g", @@ -1101,7 +1412,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cb34sfl6psfiqtovhpt4 "Content-type": "application/json" }, "method": "POST", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Deleted ratio: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{ lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0)}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Deleted ratio: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" }, "sub_type": "", "enabled": true @@ -1135,314 +1446,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cb34sfl6psfiqtovhpt4 }, "recovery_notification_config": { "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", - "normal": [ - { - "id": "cj8bq8d3q95ogankugqg", - "enabled": true - }, - { - "id": "cj8ctat3q95l9ebbntlg", - "enabled": true - }, - { - "id": "cj8atf53q95lhahebg8g", - "enabled": true - }, - { - "id": "cj8e9s53q95gsdbb054g", - "enabled": true - }, - { - "id": "cj8e9gt3q95gsdbb0170", - "enabled": true - }, - { - "id": "cj86l0l3q95rrpfea6ug", - "enabled": true - } - ], - "event_enabled": true - }, - "schedule": { - "interval": "1m" - }, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - } -} - -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp20n2anisjmu4gehc5 -{ - "id": "builtin-cbp20n2anisjmu4gehc5", - "created": "2022-08-09T08:52:44.63345561Z", - "updated": "2023-08-09T09:43:37.945659792+08:00", - "name": "Elasticsearch node left cluster", - "enabled": true, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - ".infini_node" - ], - "filter": {}, - "raw_filter": { - "match_phrase": { - "metadata.labels.status": "unavailable" - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.cluster_id", - "limit": 5 - }, - { - "field": "metadata.node_id", - "limit": 50 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "metadata.labels.status", - "statistic": "count" - } - ], - "format_type": "num", - "bucket_label": { - "enabled": false - }, - "expression": "count(metadata.labels.status)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "1" - ], - "priority": "critical" - } - ] - }, - "notification_config": { - "enabled": true, - "title": "Elasticsearch node left cluster", - "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), Left: {{.result_value}}\n{{end}}", - "normal": [ - { - "id": "cgnb2nt3q95nmusjl65g", - "created": "2023-04-06T11:47:43.104108279Z", - "updated": "2023-08-07T10:42:17.686776304+08:00", - "name": "Slack Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-type": "application/json" - }, - "method": "POST", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Left: {{.result_value}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" - }, - "sub_type": "slack", - "enabled": true - }, - { - "id": "cgnb2r53q95nmusjl6vg", - "enabled": true - }, - { - "id": "ch1os6t3q95lk6lepkq0", - "enabled": true - }, - { - "id": "cgnb2kt3q95nmusjl64g", - "enabled": true - }, - { - "id": "cj865st3q95rega919ig", - "enabled": true - }, - { - "id": "cgiospt3q95q49k3u00g", - "enabled": true - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "recovery_notification_config": { - "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", - "normal": [ - { - "id": "cj8bq8d3q95ogankugqg", - "enabled": true - }, - { - "id": "cj8ctat3q95l9ebbntlg", - "enabled": true - }, - { - "id": "cj8e9s53q95gsdbb054g", - "enabled": true - }, - { - "id": "cj8e9gt3q95gsdbb0170", - "enabled": true - }, - { - "id": "cj86l0l3q95rrpfea6ug", - "enabled": true - }, - { - "id": "cj8atf53q95lhahebg8g", - "enabled": true - } - ], - "event_enabled": true - }, - "schedule": { - "interval": "1m" - }, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - } -} -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calavvp7h710dpnp32r3 -{ - "id": "builtin-calavvp7h710dpnp32r3", - "created": "2022-06-16T04:22:23.001354546Z", - "updated": "2023-08-09T09:43:58.551403706+08:00", - "name": "Index Health Change to Red", - "enabled": true, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - ".infini_index" - ], - "filter": {}, - "raw_filter": { - "match": { - "metadata.labels.health_status": "red" - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.cluster_id", - "limit": 50 - }, - { - "field": "metadata.index_name", - "limit": 1000 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "metadata.index_name", - "statistic": "count" - } - ], - "format_type": "num", - "bucket_label": { - "enabled": false - }, - "expression": "count(metadata.index_name)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "1" - ], - "priority": "high" - } - ] - }, - "notification_config": { - "enabled": true, - "title": "Health of Indices ({{len .results}} indices in total) Changed to Red", - "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}) is Red now\n{{end}}", - "normal": [ - { - "id": "cgnb2nt3q95nmusjl65g", - "created": "2023-04-06T11:47:43.104108279Z", - "updated": "2023-08-07T15:17:26.18861218+08:00", - "name": "[Alerting] Slack Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}> is Red now\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" - }, - "sub_type": "slack", - "enabled": true - }, - { - "id": "cgiospt3q95q49k3u00g", - "enabled": true - }, - { - "id": "cj865st3q95rega919ig", - "enabled": true - }, - { - "id": "cgnb2r53q95nmusjl6vg", - "enabled": true - }, - { - "id": "ch1os6t3q95lk6lepkq0", - "enabled": true - }, - { - "id": "cgnb2kt3q95nmusjl64g", - "enabled": true - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "recovery_notification_config": { - "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", "normal": [ { "id": "cj8bq8d3q95ogankugqg", @@ -1483,7 +1488,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7 { "id": "builtin-cbp2e4ianisjmu4giqs7", "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2023-08-09T09:44:31.495696286+08:00", + "updated": "2023-08-09T22:39:15.339913317+08:00", "name": "Search latency is great than 500ms", "enabled": true, "resource": { @@ -1583,7 +1588,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7 "notification_config": { "enabled": true, "title": "Search latency is great than 500ms", - "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}), Latency: {{.result_value | to_fixed 2}}ms\n{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nIndex: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Latency: {{.result_value | to_fixed 2}}ms\n{{end}}", "normal": [ { "id": "cgnb2nt3q95nmusjl65g", @@ -1596,8 +1601,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7 "Content-type": "application/json" }, "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "\n{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Search latency: {{.result_value | to_fixed 2}}ms\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + "body": "\n{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{ lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0)}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Search latency: {{.result_value | to_fixed 2}}ms\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" }, "sub_type": "slack", "enabled": true @@ -1631,8 +1635,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7 }, "recovery_notification_config": { "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", "normal": [ { "id": "cj8bq8d3q95ogankugqg", @@ -1673,7 +1677,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8 { "id": "builtin-calaqnh7h710dpnp2bm8", "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2023-08-09T09:46:34.428920151+08:00", + "updated": "2023-08-09T22:38:55.677122718+08:00", "name": "JVM utilization is Too High", "enabled": true, "resource": { @@ -1767,7 +1771,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8 "notification_config": { "enabled": true, "title": "JVM Usage of Nodes ({{len .results}} nodes in total) >= {{.first_threshold}}%", - "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), JVM Usage: {{.result_value | to_fixed 2}}%\n{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nNode: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), JVM Usage: {{.result_value | to_fixed 2}}%\n{{end}}", "normal": [ { "id": "cgnb2nt3q95nmusjl65g", @@ -1780,7 +1784,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8 "Content-type": "application/json" }, "method": "POST", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, JVM Usage: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, JVM Usage: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" }, "sub_type": "slack", "enabled": true @@ -1814,8 +1818,8 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8 }, "recovery_notification_config": { "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", "normal": [ { "id": "cj8bq8d3q95ogankugqg", @@ -1856,7 +1860,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2 { "id": "builtin-calakp97h710dpnp1fa2", "created": "2022-06-16T03:58:29.437447113Z", - "updated": "2023-08-09T09:42:57.901272952+08:00", + "updated": "2023-08-09T22:33:25.692835454+08:00", "name": "CPU utilization is Too High", "enabled": true, "resource": { @@ -1925,7 +1929,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2 "minimum_period_match": 1, "operator": "gte", "values": [ - "85" + "80" ], "priority": "low" }, @@ -1950,7 +1954,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2 "notification_config": { "enabled": true, "title": "CPU Usage of Nodes ({{len .results}} nodes in total) >= {{.first_threshold}}%", - "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), CPU Usage: {{.result_value | to_fixed 2}}%\n{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nNode: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), CPU Usage: {{.result_value | to_fixed 2}}%\n{{end}}", "normal": [ { "id": "cgnb2nt3q95nmusjl65g", @@ -1963,8 +1967,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2 "Content-type": "application/json" }, "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, CPU Usage: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, CPU Usage: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" }, "sub_type": "slack", "enabled": true @@ -1998,8 +2001,162 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2 }, "recovery_notification_config": { "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6 +{ + "id": "builtin-calgapp7h710dpnpbeb6", + "created": "2022-06-16T10:26:47.360988761Z", + "updated": "2023-08-09T22:37:44.038127695+08:00", + "name": "Shard Storage >= 55G", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "range": { + "payload.elasticsearch.index_stats.shard_info.store_in_bytes": { + "gte": 59055800320 + } + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.index_name", + "limit": 500 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.index_stats.shard_info.store_in_bytes", + "statistic": "max" + } + ], + "format_type": "bytes", + "bucket_label": { + "enabled": false + }, + "expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "59055800320" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Shard Storage >55GB in ({{len .results}} indices in total)", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nIndex: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Max Shard Storage: {{.result_value | format_bytes 2}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T14:02:53.734855705+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{ lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0)}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Max shard storage: {{.result_value | format_bytes 2}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "šŸŒˆ [{{.rule_name}}] Resolved", + "message": "- EventID: {{.event_id}}\n- Target: {{.resource_name}}-{{.objects}}\n- TriggerAt: {{.trigger_at | datetime}}\n- ResolveAt: {{.timestamp | datetime}}\n- Duration: {{.duration}}", "normal": [ { "id": "cj8bq8d3q95ogankugqg", @@ -2040,7 +2197,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnogps1 { "id": "builtin-cal8n7p7h710dpnogps1", "created": "2022-06-16T03:11:01.445958361Z", - "updated": "2023-08-09T09:43:16.31964237+08:00", + "updated": "2023-08-10T17:16:34.900352415+08:00", "name": "Disk utilization is Too High", "enabled": true, "resource": { @@ -2139,7 +2296,7 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnogps1 "notification_config": { "enabled": true, "title": "Disk Usage of Nodes ({{len .results}} nodes in total) >= {{.first_threshold}}%", - "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), Usage: {{.result_value | to_fixed 2}}% / Free: {{.relation_values.b | format_bytes 2}}\n{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nNode: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Usage: {{.result_value | to_fixed 2}}% / Free: {{.relation_values.b | format_bytes 2}}\n{{end}}", "normal": [ { "id": "cgnb2nt3q95nmusjl65g", @@ -2184,175 +2341,9 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnogps1 }, "recovery_notification_config": { "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", - "normal": [ - { - "id": "cj8bq8d3q95ogankugqg", - "enabled": true - }, - { - "id": "cj8ctat3q95l9ebbntlg", - "enabled": true - }, - { - "id": "cj8atf53q95lhahebg8g", - "enabled": true - }, - { - "id": "cj8e9s53q95gsdbb054g", - "enabled": true - }, - { - "id": "cj8e9gt3q95gsdbb0170", - "enabled": true - }, - { - "id": "cj86l0l3q95rrpfea6ug", - "enabled": true - } - ], - "event_enabled": true - }, - "schedule": { - "interval": "1m" - }, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - } -} -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0 -{ - "id": "builtin-cal8n7p7h710dpnoaps0", - "created": "2022-06-16T01:47:11.326727124Z", - "updated": "2023-08-09T09:50:05.833535441+08:00", - "name": "Cluster Health Change to Red", - "enabled": true, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - ".infini_metrics*" - ], - "filter": {}, - "raw_filter": { - "bool": { - "must": [ - { - "match": { - "payload.elasticsearch.cluster_health.status": "red" - } - }, - { - "term": { - "metadata.name": { - "value": "cluster_health" - } - } - } - ] - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.cluster_health.status", - "statistic": "count" - } - ], - "format_type": "num", - "bucket_label": { - "enabled": false - }, - "expression": "count(payload.elasticsearch.cluster_health.status)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "1" - ], - "priority": "critical" - } - ] - }, - "notification_config": { - "enabled": true, - "title": "Health of Clusters ({{len .results}} clusters in total) Changed to Red", - "message": "{{range .results}}\nCluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/) is Red now\n{{end}}", - "normal": [ - { - "id": "cgnb2nt3q95nmusjl65g", - "created": "2023-04-06T11:47:43.104108279Z", - "updated": "2023-08-07T15:02:17.165625799+08:00", - "name": "[Alerting] Slack Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Cluster:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}> is Red now\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" - }, - "sub_type": "slack", - "enabled": true - }, - { - "id": "cgiospt3q95q49k3u00g", - "enabled": true - }, - { - "id": "cj865st3q95rega919ig", - "enabled": true - }, - { - "id": "cgnb2r53q95nmusjl6vg", - "enabled": true - }, - { - "id": "ch1os6t3q95lk6lepkq0", - "enabled": true - }, - { - "id": "cgnb2kt3q95nmusjl64g", - "enabled": true - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "recovery_notification_config": { - "enabled": true, - "title": "Alert [{{.rule_name}}] Resolved", + "title": "šŸŒˆ [{{.rule_name}}] Resolved", "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", "normal": [ - { - "id": "cj8atf53q95lhahebg8g", - "enabled": true - }, { "id": "cj8bq8d3q95ogankugqg", "enabled": true @@ -2361,6 +2352,10 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0 "id": "cj8ctat3q95l9ebbntlg", "enabled": true }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, { "id": "cj8e9s53q95gsdbb054g", "enabled": true