diff --git a/config/initialization.tpl b/config/initialization.tpl index 297ee9ad..3f349dcd 100644 --- a/config/initialization.tpl +++ b/config/initialization.tpl @@ -601,1157 +601,1767 @@ PUT $[[INDEX_PREFIX]]activities-00001 } } - -#alerting -POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calakp97h710dpnp1fa2 -{ - "id": "builtin-calakp97h710dpnp1fa2", - "created": "2022-06-16T03:58:29.437447113Z", - "updated": "2022-07-21T23:12:51.111569117Z", - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "name": "CPU utilization is Too High", - "enabled": false, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]metrics*" - ], - "filter": {}, - "raw_filter": { - "bool": { - "must": [ - { - "term": { - "metadata.name": { - "value": "node_stats" - } - } - }, - { - "term": { - "metadata.category": { - "value": "elasticsearch" - } - } - } - ] - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.node_id", - "limit": 300 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.node_stats.process.cpu.percent", - "statistic": "avg" - } - ], - "format_type": "ratio", - "expression": "avg(payload.elasticsearch.node_stats.process.cpu.percent)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "80" - ], - "priority": "low" - }, - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "90" - ], - "priority": "medium" - }, - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "95" - ], - "priority": "high" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "CPU Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%", - "message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \nCPU:{{.result_value | to_fixed 2}}%;\n{{end}}", - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "6h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - +#alerting channel #The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnoaps0 +POST $[[INDEX_PREFIX]]channel/_doc/cj865st3q95rega919ig { - "id": "builtin-cal8n7p7h710dpnoaps0", - "created": "2022-06-16T01:47:11.326727124Z", - "updated": "2022-07-13T04:00:06.181994982Z", - "name": "Cluster Health Change to Red", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]metrics*" - ], - "filter": {}, - "raw_filter": { - "bool": { - "must": [ - { - "match": { - "payload.elasticsearch.cluster_health.status": "red" - } - }, - { - "term": { - "metadata.name": { - "value": "cluster_health" - } - } - } - ] - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.cluster_health.status", - "statistic": "count" - } - ], - "format_type": "num", - "expression": "count(payload.elasticsearch.cluster_health.status)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "1" - ], - "priority": "critical" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "Health of Cluster[s] ({{.first_group_value}} ..., {{len .results}} clusters in total) Changed to Red", - "message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}, Name:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }} is RED now;\n{{end}}", - "normal": [ - { - "created": "2022-06-16T01:47:11.326727124Z", - "updated": "2022-06-16T01:47:11.326727124Z", - "name": "Slack webhook", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|View Cluster Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - }, - { - "created": "2022-06-16T01:47:11.326727124Z", - "updated": "2022-06-16T01:47:11.326727124Z", - "name": "DingTalk", - "type": "webhook", - "sub_type": "dingtalk", - "enabled": true, - "webhook": { - "header_params": { - "Content-type": "application/json" - }, - "method": "POST", - "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", - "body": "{\"msgtype\": \"text\",\"text\": {\"content\":\"Alerting: \\n{{.title}}\\n\\n{{.message}}\\nLink:{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"}}" - } - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnogps1 -{ - "id": "builtin-cal8n7p7h710dpnogps1", - "created": "2022-06-16T03:11:01.445958361Z", - "updated": "2022-07-22T00:06:26.498903821Z", - "name": "Disk utilization is Too High", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]metrics*" - ], - "filter": {}, - "raw_filter": { - "bool": { - "must": [ - { - "term": { - "metadata.name": { - "value": "node_stats" - } - } - }, - { - "term": { - "metadata.category": { - "value": "elasticsearch" - } - } - } - ] - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.node_id", - "limit": 200 - } - ], - "formula": "((a-b)/a)*100", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.node_stats.fs.data.total_in_bytes", - "statistic": "max" - }, - { - "name": "b", - "field": "payload.elasticsearch.node_stats.fs.data.free_in_bytes", - "statistic": "max" - } - ], - "format_type": "ratio", - "expression": "((max(payload.elasticsearch.node_stats.fs.data.total_in_bytes)-max(payload.elasticsearch.node_stats.fs.data.free_in_bytes))/max(payload.elasticsearch.node_stats.fs.data.total_in_bytes))*100" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 5, - "operator": "gte", - "values": [ - "85" - ], - "priority": "low" - }, - { - "minimum_period_match": 5, - "operator": "gte", - "values": [ - "90" - ], - "priority": "medium" - }, - { - "minimum_period_match": 5, - "operator": "gte", - "values": [ - "95" - ], - "priority": "high" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "Disk Utilization is Too High", - "message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID：{{index .group_values 0}} ;\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID：{{index .group_values 1}} ;\nDisk Usage:{{.result_value | to_fixed 2}}%；Free Storage:{{.relation_values.b | format_bytes 2}}；\n{{end}}", - "normal": [ - { - "created": "0001-01-01T00:00:00Z", - "updated": "0001-01-01T00:00:00Z", - "name": "Slack", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Free:* {{.relation_values.b | format_bytes 2}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "3h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cbp20n2anisjmu4gehc5 -{ - "id": "builtin-cbp20n2anisjmu4gehc5", - "created": "2022-08-09T08:52:44.63345561Z", - "updated": "2022-08-09T08:52:44.633455664Z", - "name": "Elasticsearch node left cluster", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]node" - ], - "filter": {}, - "raw_filter": { - "match_phrase": { - "metadata.labels.status": "unavailable" - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.cluster_id", - "limit": 5 - }, - { - "field": "metadata.node_id", - "limit": 50 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "metadata.labels.status", - "statistic": "count" - } - ], - "format_type": "num", - "expression": "count(metadata.labels.status)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "1" - ], - "priority": "critical" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "Elasticsearch node left cluster", - "message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \n{{end}}", - "normal": [ - { - "created": "2022-08-09T08:52:44.63345561Z", - "updated": "2022-08-09T08:52:44.63345561Z", - "name": "Wechat", - "type": "webhook", - "sub_type": "wechat", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.WECHAT_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n{{.title}}\\n\n {{range .results}}\n >ClusterID:{{index .group_values 0}}\n >NodeID:{{index .group_values 1}}\n >Priority:{{.priority}}\n >Link:[View Cluster Monitoring]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}) \n {{end}}\"\n }\n}\n" - } - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calavvp7h710dpnp32r3 -{ - "id": "builtin-calavvp7h710dpnp32r3", - "created": "2022-06-16T04:22:23.001354546Z", - "updated": "2022-07-21T23:10:36.70696738Z", - "name": "Index Health Change to Red", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]index" - ], - "filter": {}, - "raw_filter": { - "match_phrase": { - "metadata.labels.health_status": "red" - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.cluster_id", - "limit": 5 - }, - { - "field": "metadata.index_name", - "limit": 5 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "metadata.index_name", - "statistic": "count" - } - ], - "format_type": "num", - "expression": "count(metadata.index_name)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "1" - ], - "priority": "high" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "Health of Indices ({{.first_group_value}} ..., {{len .results}} indices in total) Changed to Red", - "message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex name:{{index .group_values 1}}; {{end}}", - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}?_g=%7B%22tab%22%3A%22indices%22%7D|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calaqnh7h710dpnp2bm8 -{ - "id": "builtin-calaqnh7h710dpnp2bm8", - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-07-21T23:12:07.142532243Z", - "name": "JVM utilization is Too High", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]metrics*" - ], - "filter": {}, - "raw_filter": { - "bool": { - "must": [ - { - "term": { - "metadata.name": { - "value": "node_stats" - } - } - }, - { - "term": { - "metadata.category": { - "value": "elasticsearch" - } - } - } - ] - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.node_id", - "limit": 300 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.node_stats.jvm.mem.heap_used_percent", - "statistic": "p90" - } - ], - "format_type": "ratio", - "expression": "p90(payload.elasticsearch.node_stats.jvm.mem.heap_used_percent)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "80" - ], - "priority": "low" - }, - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "90" - ], - "priority": "medium" - }, - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "95" - ], - "priority": "high" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "JVM Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%", - "message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNode name:{{index .group_values 1}}; memory used percent：{{.result_value | to_fixed 2}}%;{{end}}", - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "3h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cbp2e4ianisjmu4giqs7 -{ - "id": "builtin-cbp2e4ianisjmu4giqs7", - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-08-09T09:39:29.604751601Z", - "name": "Search latency is great than 500ms", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]metrics*" - ], - "filter": {}, - "raw_filter": { - "bool": { - "must": [ - { - "term": { - "metadata.name": { - "value": "index_stats" - } - } - }, - { - "term": { - "metadata.category": { - "value": "elasticsearch" - } - } - } - ], - "must_not": [ - { - "term": { - "metadata.labels.index_name": { - "value": "_all" - } - } - } - ] - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.index_name", - "limit": 500 - } - ], - "formula": "a/b", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.index_stats.total.search.query_time_in_millis", - "statistic": "rate" - }, - { - "name": "b", - "field": "payload.elasticsearch.index_stats.primaries.search.query_total", - "statistic": "rate" - } - ], - "format_type": "num", - "expression": "rate(payload.elasticsearch.index_stats.total.search.query_time_in_millis)/rate(payload.elasticsearch.index_stats.primaries.search.query_total)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "500" - ], - "priority": "medium" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "Search latency is great than 500ms", - "message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex Name:{{index .group_values 1}}; \nCurrent Value:{{.result_value | to_fixed 2}}ms;\n{{end}}", - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Latency:* {{.result_value | to_fixed 2}}ms\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calgapp7h710dpnpbeb6 -{ - "id": "builtin-calgapp7h710dpnpbeb6", - "created": "2022-06-16T10:26:47.360988761Z", - "updated": "2022-07-22T00:03:34.044562893Z", - "name": "Shard Storage >= 55G", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]metrics*" - ], - "filter": {}, - "raw_filter": { - "range": { - "payload.elasticsearch.index_stats.shard_info.store_in_bytes": { - "gte": 59055800320 - } - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.index_name", - "limit": 500 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.index_stats.shard_info.store_in_bytes", - "statistic": "max" - } - ], - "format_type": "bytes", - "expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "53687091200" - ], - "priority": "high" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "Shard Storage >55GB in ({{.first_group_value}} ..., {{len .results}} indices in total)", - "message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }};\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22});\nMax Shard Storage：{{.result_value | format_bytes 2}};\n{{end}}", - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Max Shard Storage:* {{.result_value | format_bytes 2}}\"\n },\n \n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "24h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/_doc/cb34sfl6psfiqtovhpt4 -{ - "id": "cb34sfl6psfiqtovhpt4", - "created": "2022-07-07T03:08:46.297166036Z", - "updated": "2022-08-09T08:40:05.323148338Z", - "name": "Too Many Deleted Documents", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]metrics*" - ], - "filter": {}, - "raw_filter": { - "range": { - "payload.elasticsearch.cluster_stats.indices.store.size_in_bytes": { - "gte": 32212254720 - } - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.index_name", - "limit": 300 - } - ], - "formula": "(a/(a+b))*100", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.index_stats.primaries.docs.deleted", - "statistic": "max" - }, - { - "name": "b", - "field": "payload.elasticsearch.index_stats.primaries.docs.count", - "statistic": "max" - } - ], - "format_type": "ratio", - "expression": "(max(payload.elasticsearch.index_stats.primaries.docs.deleted)/(max(payload.elasticsearch.index_stats.primaries.docs.deleted)+max(payload.elasticsearch.index_stats.primaries.docs.count)))*100" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "30" - ], - "priority": "medium" - }, - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "40" - ], - "priority": "high" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "Too Many Deleted Documents (>30%)", - "message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex:{{index .group_values 0}}; \nRatio of Deleted Documents:{{.result_value}};\n{{end}}", - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Deleted:* {{.result_value | to_fixed 2}}%\"\n },\n \n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "24h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]channel/_doc/builtin-cgnb2nt3q95nmusjl65g -{ - "id": "builtin-cgnb2nt3q95nmusjl65g", - "created": "2023-04-06T11:47:43.104108279Z", - "updated": "2023-08-04T10:34:29.112776+08:00", - "name": "Slack Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json", - "Content-type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"【Demo】Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"high\"}} \"#EB4C21\" {{else if eq .priority \"medium\"}} \"#FFB449\" {{else if eq .priority \"low\"}} \"#87d068\" {{else}} \"#2db7f5\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{index .group_values 0 | lookup \"category=metadata, object=cluster, property=name, default=N/A\"}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|View Cluster Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - }, - "sub_type": "slack" -} -POST $[[INDEX_PREFIX]]channel/_doc/builtin-cgiospt3q95q49k3u00g -{ - "id": "builtin-cgiospt3q95q49k3u00g", - "created": "2023-03-30T13:28:07.531263747Z", - "updated": "2023-08-04T11:13:51.608186+08:00", - "name": "DingTalk", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json", - "Content-type": "application/json" - }, - "method": "POST", - "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", - "body": "{\"msgtype\": \"text\",\"text\": {\"content\":\"------------------------------------\\n【 INFINI Platform Alerting 】\\n{{.title}}\\n------------------------------------\\n{{.message}}\\nLink:{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"}}" - }, - "sub_type": "dingtalk" -} -POST $[[INDEX_PREFIX]]channel/_doc/builtin-ch1os6t3q95lk6lepkq0 -{ - "id": "builtin-ch1os6t3q95lk6lepkq0", - "created": "2023-04-22T07:34:51.848540351Z", - "updated": "2023-08-04T10:34:13.937983+08:00", - "name": "Feishu Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json", - "Content-type": "application/json" - }, - "method": "POST", - "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msg_type\": \"text\",\n \"content\": \"{\\\"text\\\":\\\"Alerting: {{.title}} \\\\n Link:{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\\\"}\"\n}" - }, - "sub_type": "feishu" -} -POST $[[INDEX_PREFIX]]channel/_doc/builtin-cgnb2kt3q95nmusjl64g -{ - "id": "builtin-cgnb2kt3q95nmusjl64g", - "created": "2023-04-06T11:47:31.161587662Z", - "updated": "2023-08-04T10:33:54.594583+08:00", - "name": "Wechat Notification", + "id": "cj865st3q95rega919ig", + "created": "2023-08-07T11:20:19.223545026+08:00", + "updated": "2023-08-08T18:42:26.506499014+08:00", + "name": "[Alerting] Discord", "type": "webhook", "webhook": { "header_params": { "Content-Type": "application/json" }, "method": "POST", - "url": "{{$.env.WECHAT_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n{{.title}}\\n{{.message}}\"\n }\n}" + "url": "{{$.env.DISCORD_WEBHOOK_ENDPOINT}}", + "body": "{\n \"content\": \"Hello Alerting\"\n}" }, - "sub_type": "wechat" + "sub_type": "discord", + "enabled": false } -POST $[[INDEX_PREFIX]]channel/_doc/builtin-cgnb2r53q95nmusjl6vg +POST $[[INDEX_PREFIX]]channel/_doc/cj86l0l3q95rrpfea6ug { - "id": "builtin-cgnb2r53q95nmusjl6vg", + "id": "cj86l0l3q95rrpfea6ug", + "created": "2023-08-07T11:52:34.192522006+08:00", + "updated": "2023-08-08T18:42:30.162079286+08:00", + "name": "[Recovery] Discord\t", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DISCORD_WEBHOOK_ENDPOINT}}", + "body": "{\n\n}" + }, + "sub_type": "discord", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/_doc/cgnb2nt3q95nmusjl65g +{ + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-08T22:19:08.601341574+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing !*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.trigger_at | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Cluster:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/_doc/cj8bq8d3q95ogankugqg +{ + "id": "cj8bq8d3q95ogankugqg", + "created": "2023-08-07T17:45:05.534408059+08:00", + "updated": "2023-08-08T19:26:34.009668892+08:00", + "name": "[Recovery] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*:rainbow: Alert [{{.rule_name}}] Resolved*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*ResolveAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Target:* {{.resource_name}}-{{.objects}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.trigger_at | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Duration:* {{.duration}}\"\n }\n },\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n }\n ]\n}" + }, + "sub_type": "slack", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/_doc/cgiospt3q95q49k3u00g +{ + "id": "cgiospt3q95q49k3u00g", + "created": "2023-03-30T13:28:07.531263747Z", + "updated": "2023-08-08T22:19:07.545051029+08:00", + "name": "[Alerting] DingTalk Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.com/img/email/alert-header.png)\\n\\n🔥 **{{.title}}**\\n\\nIncident [{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing !\\n\\nPriority: {{.priority}}\\n\\nEventID: {{.event_id}}\\n\\nTarget: {{.resource_name}}-{{.objects}}\\n\\nTriggerAt: {{.trigger_at | datetime}}\\n\\n---\\n\\n{{.message}}\"\n }\n}" + }, + "sub_type": "dingtalk", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/_doc/cgnb2r53q95nmusjl6vg +{ + "id": "cgnb2r53q95nmusjl6vg", "created": "2023-04-06T11:47:56.652637309Z", - "updated": "2023-08-04T10:12:44.675016+08:00", - "name": "SMS Notification", + "updated": "2023-08-08T19:49:20.312590885+08:00", + "name": "[Alerting] Email Notification", "type": "email", "sub_type": "email", "email": { "server_id": "", "recipients": { - "to": [] + "to": [], + "cc": [] }, - "subject": "{{.title}}", - "body": "Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\n{{.message}}" + "subject": "[INFINI Platform Alerting] 🔥 {{.title}}", + "body": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n
\n \"email-header\"\n
\n
\n \n \n \n \n \n \n
\n
\n
\n {{.title}}\n

\n \n

Priority: {{.priority}}

\n

EventID: {{.event_id}}

\n

Target: {{.resource_name}}-{{.objects}}

\n

TriggerAt: {{.trigger_at | datetime}}

\n {{.message | md_to_html}}\n
\n

\n \n \n View Detail\n \n

\n \n \n \n

\n \n

\n \"INFINI\n
\n
\n \n
\n

\n \n \n
\n
\n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n
\n
\n \n \n \n
\n
\n \n
\n \n \n
\n
\n \n ", + "content_type": "text/html" + }, + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/_doc/ch1os6t3q95lk6lepkq0 +{ + "id": "ch1os6t3q95lk6lepkq0", + "created": "2023-04-22T07:34:51.848540351Z", + "updated": "2023-08-09T09:29:26.412223281+08:00", + "name": "[Alerting] Feishu Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI Platform Alerting ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"{{if eq .priority \"critical\"}}red{{else if eq .priority \"high\"}}orange{{else if eq .priority \"medium\"}}yellow{{else if eq .priority \"low\"}}grey{{else}}blue{{end}}\"\n },\n \"elements\": [{\n \"tag\": \"markdown\",\n \"content\": \"🔥 Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing ! \\n **{{.title}}**\\nPriority: {{.priority}}\\nEventID: {{.event_id}}\\nTarget: {{.resource_name}}-{{.objects}}\\nTriggerAt: {{.trigger_at | datetime}}\"\n },{\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n }\n ]\n}\n}" + }, + "sub_type": "feishu", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/_doc/cj8e9s53q95gsdbb054g +{ + "id": "cj8e9s53q95gsdbb054g", + "created": "2023-08-07T20:34:56.334695598+08:00", + "updated": "2023-08-08T21:34:50.261294305+08:00", + "name": "[Recovery] Feishu Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI Platform Alerting ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"green\"\n },\n \"elements\": [\n {\n \"tag\": \"markdown\",\n \"content\": \"🌈 **{{.title}}**\"\n },\n {\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n }\n ]\n }\n}" + }, + "sub_type": "feishu", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/_doc/cj8ctat3q95l9ebbntlg + { + "id": "cj8ctat3q95l9ebbntlg", + "created": "2023-08-07T18:59:55.28732241+08:00", + "updated": "2023-08-08T19:46:30.557046793+08:00", + "name": "[Recovery] DingTalk Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.com/img/email/recovery-header.png)\\n\\n🌈 **{{.title}}**\\n\\n{{.message}}\\n\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n }\n}\n" + }, + "sub_type": "dingtalk", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/_doc/cgnb2kt3q95nmusjl64g +{ + "id": "cgnb2kt3q95nmusjl64g", + "created": "2023-04-06T11:47:31.161587662Z", + "updated": "2023-08-08T22:19:06.712911427+08:00", + "name": "[Alerting] Wechat Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.WECOM_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"**[ INFINI Platform Alerting ]**\\n🔥 Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing !\\n**{{.title}}**\\nPriority: {{.priority}}\\n\\nEventID: {{.event_id}}\\n\\nTarget: {{.resource_name}}-{{.objects}}\\n\\nTriggerAt: {{.trigger_at | datetime}}\\n{{.message}}\"\n }\n}" + }, + "sub_type": "wechat", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/_doc/cj8e9gt3q95gsdbb0170 +{ + "id": "cj8e9gt3q95gsdbb0170", + "created": "2023-08-07T20:34:11.998953512+08:00", + "updated": "2023-08-08T19:47:08.270014715+08:00", + "name": "[Recovery] Wechat Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.WECOM_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"**[ INFINI Platform Alerting ]**\\n🌈 **{{.title}}**\\n\\n{{.message}}\\n\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n }\n}\n" + }, + "sub_type": "wechat", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/_doc/cj8atf53q95lhahebg8g +{ + "id": "cj8atf53q95lhahebg8g", + "created": "2023-08-07T16:43:40.062389175+08:00", + "updated": "2023-08-08T19:50:15.803258835+08:00", + "name": "[Recovery] Email Notification", + "type": "email", + "sub_type": "email", + "email": { + "server_id": "", + "recipients": { + "to": [], + "cc": [] + }, + "subject": "[INFINI Platform Alerting] 🌈 {{.title}}", + "body": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n
\n \"email-header\"\n
\n
\n \n \n \n \n \n \n
\n
\n
\n 🌈 {{.title}}\n

\n {{.message | md_to_html}}\n
\n

\n \n \n View Detail\n \n

\n \n \n \n

\n \n

\n \"INFINI\n
\n
\n \n
\n

\n \n \n
\n
\n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n
\n
\n \n \n \n
\n
\n \n
\n \n \n
\n
\n \n ", + "content_type": "text/html" + }, + "enabled": false +} + +#alerting +POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calgapp7h710dpnpbeb6 +{ + "id": "builtin-calgapp7h710dpnpbeb6", + "created": "2022-06-16T10:26:47.360988761Z", + "updated": "2023-08-09T09:44:58.584645596+08:00", + "name": "Shard Storage >= 55G", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "range": { + "payload.elasticsearch.index_stats.shard_info.store_in_bytes": { + "gte": 59055800320 + } + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.index_name", + "limit": 500 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.index_stats.shard_info.store_in_bytes", + "statistic": "max" + } + ], + "format_type": "bytes", + "bucket_label": { + "enabled": false + }, + "expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "59055800320" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Shard Storage >55GB in ({{len .results}} indices in total)", + "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}), Max Shard Storage: {{.result_value | format_bytes 2}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T14:02:53.734855705+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Max shard storage: {{.result_value | format_bytes 2}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "- EventID: {{.event_id}}\n- Target: {{.resource_name}}-{{.objects}}\n- TriggerAt: {{.trigger_at}}\n- ResolveAt: {{.timestamp | datetime}}\n- Duration: {{.duration}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cb34sfl6psfiqtovhpt4 +{ + "id": "builtin-cb34sfl6psfiqtovhpt4", + "created": "2022-07-07T03:08:46.297166036Z", + "updated": "2023-08-09T09:45:34.123901475+08:00", + "name": "Too Many Deleted Documents", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "range": { + "payload.elasticsearch.cluster_stats.indices.store.size_in_bytes": { + "gte": 32212254720 + } + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 20 + }, + { + "field": "metadata.labels.index_name", + "limit": 10 + } + ], + "formula": "(a/(a+b))*100", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.index_stats.primaries.docs.deleted", + "statistic": "max" + }, + { + "name": "b", + "field": "payload.elasticsearch.index_stats.primaries.docs.count", + "statistic": "max" + } + ], + "format_type": "ratio", + "bucket_label": { + "enabled": false + }, + "expression": "(max(payload.elasticsearch.index_stats.primaries.docs.deleted)/(max(payload.elasticsearch.index_stats.primaries.docs.deleted)+max(payload.elasticsearch.index_stats.primaries.docs.count)))*100" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "30" + ], + "priority": "medium" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "40" + ], + "priority": "high" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "55" + ], + "priority": "low" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Too Many Deleted Documents (>30%)", + "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), Deleted: {{.result_value | to_fixed 2}}%\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "name": "", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Deleted ratio: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "24h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} + +POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cbp20n2anisjmu4gehc5 +{ + "id": "builtin-cbp20n2anisjmu4gehc5", + "created": "2022-08-09T08:52:44.63345561Z", + "updated": "2023-08-09T09:43:37.945659792+08:00", + "name": "Elasticsearch node left cluster", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_node" + ], + "filter": {}, + "raw_filter": { + "match_phrase": { + "metadata.labels.status": "unavailable" + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.cluster_id", + "limit": 5 + }, + { + "field": "metadata.node_id", + "limit": 50 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "metadata.labels.status", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(metadata.labels.status)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "critical" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Elasticsearch node left cluster", + "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), Left: {{.result_value}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T10:42:17.686776304+08:00", + "name": "Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Left: {{.result_value}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calavvp7h710dpnp32r3 +{ + "id": "builtin-calavvp7h710dpnp32r3", + "created": "2022-06-16T04:22:23.001354546Z", + "updated": "2023-08-09T09:43:58.551403706+08:00", + "name": "Index Health Change to Red", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_index" + ], + "filter": {}, + "raw_filter": { + "match": { + "metadata.labels.health_status": "red" + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.cluster_id", + "limit": 50 + }, + { + "field": "metadata.index_name", + "limit": 1000 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "metadata.index_name", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(metadata.index_name)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Health of Indices ({{len .results}} indices in total) Changed to Red", + "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}) is Red now\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T15:17:26.18861218+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}> is Red now\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cbp2e4ianisjmu4giqs7 +{ + "id": "builtin-cbp2e4ianisjmu4giqs7", + "created": "2022-06-16T04:11:10.242061032Z", + "updated": "2023-08-09T09:44:31.495696286+08:00", + "name": "Search latency is great than 500ms", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "index_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ], + "must_not": [ + { + "term": { + "metadata.labels.index_name": { + "value": "_all" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 50 + }, + { + "field": "metadata.labels.index_name", + "limit": 10 + } + ], + "formula": "a/b", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.index_stats.total.search.query_time_in_millis", + "statistic": "rate" + }, + { + "name": "b", + "field": "payload.elasticsearch.index_stats.primaries.search.query_total", + "statistic": "rate" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "rate(payload.elasticsearch.index_stats.total.search.query_time_in_millis)/rate(payload.elasticsearch.index_stats.primaries.search.query_total)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "500" + ], + "priority": "medium" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "300" + ], + "priority": "low" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Search latency is great than 500ms", + "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}), Latency: {{.result_value | to_fixed 2}}ms\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-06T15:46:34.404507399+08:00", + "name": "Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "\n{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Search latency: {{.result_value | to_fixed 2}}ms\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calaqnh7h710dpnp2bm8 +{ + "id": "builtin-calaqnh7h710dpnp2bm8", + "created": "2022-06-16T04:11:10.242061032Z", + "updated": "2023-08-09T09:46:34.428920151+08:00", + "name": "JVM utilization is Too High", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "node_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.node_id", + "limit": 300 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.node_stats.jvm.mem.heap_used_percent", + "statistic": "p90" + } + ], + "format_type": "ratio", + "bucket_label": { + "enabled": false + }, + "expression": "p90(payload.elasticsearch.node_stats.jvm.mem.heap_used_percent)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "80" + ], + "priority": "low" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "90" + ], + "priority": "medium" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "95" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "JVM Usage of Nodes ({{len .results}} nodes in total) >= {{.first_threshold}}%", + "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), JVM Usage: {{.result_value | to_fixed 2}}%\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-06T15:46:34.404507399+08:00", + "name": "Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, JVM Usage: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calakp97h710dpnp1fa2 +{ + "id": "builtin-calakp97h710dpnp1fa2", + "created": "2022-06-16T03:58:29.437447113Z", + "updated": "2023-08-09T09:42:57.901272952+08:00", + "name": "CPU utilization is Too High", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "node_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.node_id", + "limit": 300 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.node_stats.process.cpu.percent", + "statistic": "avg" + } + ], + "format_type": "ratio", + "bucket_label": { + "enabled": false + }, + "expression": "avg(payload.elasticsearch.node_stats.process.cpu.percent)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "85" + ], + "priority": "low" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "90" + ], + "priority": "medium" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "95" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "CPU Usage of Nodes ({{len .results}} nodes in total) >= {{.first_threshold}}%", + "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), CPU Usage: {{.result_value | to_fixed 2}}%\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T15:17:26.18861218+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, CPU Usage: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "6h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnogps1 +{ + "id": "builtin-cal8n7p7h710dpnogps1", + "created": "2022-06-16T03:11:01.445958361Z", + "updated": "2023-08-09T09:43:16.31964237+08:00", + "name": "Disk utilization is Too High", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "node_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.node_id", + "limit": 200 + } + ], + "formula": "((a-b)/a)*100", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.node_stats.fs.data.total_in_bytes", + "statistic": "max" + }, + { + "name": "b", + "field": "payload.elasticsearch.node_stats.fs.data.free_in_bytes", + "statistic": "max" + } + ], + "format_type": "ratio", + "bucket_label": { + "enabled": false + }, + "expression": "((max(payload.elasticsearch.node_stats.fs.data.total_in_bytes)-max(payload.elasticsearch.node_stats.fs.data.free_in_bytes))/max(payload.elasticsearch.node_stats.fs.data.total_in_bytes))*100" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 5, + "operator": "gte", + "values": [ + "80" + ], + "priority": "low" + }, + { + "minimum_period_match": 5, + "operator": "gte", + "values": [ + "90" + ], + "priority": "medium" + }, + { + "minimum_period_match": 5, + "operator": "gte", + "values": [ + "95" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Disk Usage of Nodes ({{len .results}} nodes in total) >= {{.first_threshold}}%", + "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), Usage: {{.result_value | to_fixed 2}}% / Free: {{.relation_values.b | format_bytes 2}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "name": "", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Disk Usage: {{.result_value | to_fixed 2}}%, Free: {{.relation_values.b | format_bytes 2}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n },\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Document\" \n },\n \"style\": \"primary\",\n \"url\": \"https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-cluster.html#disk-based-shard-allocation\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "6h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnoaps0 +{ + "id": "builtin-cal8n7p7h710dpnoaps0", + "created": "2022-06-16T01:47:11.326727124Z", + "updated": "2023-08-09T09:50:05.833535441+08:00", + "name": "Cluster Health Change to Red", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "match": { + "payload.elasticsearch.cluster_health.status": "red" + } + }, + { + "term": { + "metadata.name": { + "value": "cluster_health" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.cluster_health.status", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(payload.elasticsearch.cluster_health.status)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "critical" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Health of Clusters ({{len .results}} clusters in total) Changed to Red", + "message": "{{range .results}}\nCluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/) is Red now\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T15:02:17.165625799+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Cluster:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}> is Red now\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" } } diff --git a/config/initialization_v5.tpl b/config/initialization_v5.tpl index 59830774..e51cf740 100644 --- a/config/initialization_v5.tpl +++ b/config/initialization_v5.tpl @@ -546,1155 +546,1767 @@ PUT $[[INDEX_PREFIX]]activities-00001 } -#alerting -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2 -{ - "id": "builtin-calakp97h710dpnp1fa2", - "created": "2022-06-16T03:58:29.437447113Z", - "updated": "2022-07-21T23:12:51.111569117Z", - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "name": "CPU utilization is Too High", - "enabled": false, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]metrics*" - ], - "filter": {}, - "raw_filter": { - "bool": { - "must": [ - { - "term": { - "metadata.name": { - "value": "node_stats" - } - } - }, - { - "term": { - "metadata.category": { - "value": "elasticsearch" - } - } - } - ] - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.node_id", - "limit": 300 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.node_stats.process.cpu.percent", - "statistic": "avg" - } - ], - "format_type": "ratio", - "expression": "avg(payload.elasticsearch.node_stats.process.cpu.percent)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "80" - ], - "priority": "low" - }, - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "90" - ], - "priority": "medium" - }, - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "95" - ], - "priority": "high" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "CPU Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%", - "message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \nCPU:{{.result_value | to_fixed 2}}%;\n{{end}}", - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "6h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - +#alerting channel #The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0 +POST $[[INDEX_PREFIX]]channel/doc/cj865st3q95rega919ig { - "id": "builtin-cal8n7p7h710dpnoaps0", - "created": "2022-06-16T01:47:11.326727124Z", - "updated": "2022-07-13T04:00:06.181994982Z", - "name": "Cluster Health Change to Red", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]metrics*" - ], - "filter": {}, - "raw_filter": { - "bool": { - "must": [ - { - "match": { - "payload.elasticsearch.cluster_health.status": "red" - } - }, - { - "term": { - "metadata.name": { - "value": "cluster_health" - } - } - } - ] - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.cluster_health.status", - "statistic": "count" - } - ], - "format_type": "num", - "expression": "count(payload.elasticsearch.cluster_health.status)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "1" - ], - "priority": "critical" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "Health of Cluster[s] ({{.first_group_value}} ..., {{len .results}} clusters in total) Changed to Red", - "message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}, Name:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }} is RED now;\n{{end}}", - "normal": [ - { - "created": "2022-06-16T01:47:11.326727124Z", - "updated": "2022-06-16T01:47:11.326727124Z", - "name": "Slack webhook", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|View Cluster Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - }, - { - "created": "2022-06-16T01:47:11.326727124Z", - "updated": "2022-06-16T01:47:11.326727124Z", - "name": "DingTalk", - "type": "webhook", - "enabled": true, - "webhook": { - "header_params": { - "Content-type": "application/json" - }, - "method": "POST", - "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", - "body": "{\"msgtype\": \"text\",\"text\": {\"content\":\"Alerting: \\n{{.title}}\\n\\n{{.message}}\\nLink:{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"}}" - } - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnogps1 -{ - "id": "builtin-cal8n7p7h710dpnogps1", - "created": "2022-06-16T03:11:01.445958361Z", - "updated": "2022-07-22T00:06:26.498903821Z", - "name": "Disk utilization is Too High", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]metrics*" - ], - "filter": {}, - "raw_filter": { - "bool": { - "must": [ - { - "term": { - "metadata.name": { - "value": "node_stats" - } - } - }, - { - "term": { - "metadata.category": { - "value": "elasticsearch" - } - } - } - ] - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.node_id", - "limit": 200 - } - ], - "formula": "((a-b)/a)*100", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.node_stats.fs.data.total_in_bytes", - "statistic": "max" - }, - { - "name": "b", - "field": "payload.elasticsearch.node_stats.fs.data.free_in_bytes", - "statistic": "max" - } - ], - "format_type": "ratio", - "expression": "((max(payload.elasticsearch.node_stats.fs.data.total_in_bytes)-max(payload.elasticsearch.node_stats.fs.data.free_in_bytes))/max(payload.elasticsearch.node_stats.fs.data.total_in_bytes))*100" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 5, - "operator": "gte", - "values": [ - "85" - ], - "priority": "low" - }, - { - "minimum_period_match": 5, - "operator": "gte", - "values": [ - "90" - ], - "priority": "medium" - }, - { - "minimum_period_match": 5, - "operator": "gte", - "values": [ - "95" - ], - "priority": "high" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "Disk Utilization is Too High", - "message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID：{{index .group_values 0}} ;\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID：{{index .group_values 1}} ;\nDisk Usage:{{.result_value | to_fixed 2}}%；Free Storage:{{.relation_values.b | format_bytes 2}}；\n{{end}}", - "normal": [ - { - "created": "0001-01-01T00:00:00Z", - "updated": "0001-01-01T00:00:00Z", - "name": "Slack", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Free:* {{.relation_values.b | format_bytes 2}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "3h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp20n2anisjmu4gehc5 -{ - "id": "builtin-cbp20n2anisjmu4gehc5", - "created": "2022-08-09T08:52:44.63345561Z", - "updated": "2022-08-09T08:52:44.633455664Z", - "name": "Elasticsearch node left cluster", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]node" - ], - "filter": {}, - "raw_filter": { - "match_phrase": { - "metadata.labels.status": "unavailable" - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.cluster_id", - "limit": 5 - }, - { - "field": "metadata.node_id", - "limit": 50 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "metadata.labels.status", - "statistic": "count" - } - ], - "format_type": "num", - "expression": "count(metadata.labels.status)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "1" - ], - "priority": "critical" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "Elasticsearch node left cluster", - "message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \n{{end}}", - "normal": [ - { - "created": "2022-08-09T08:52:44.63345561Z", - "updated": "2022-08-09T08:52:44.63345561Z", - "name": "Wechat", - "type": "webhook", - "sub_type": "wechat", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.WECHAT_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n{{.title}}\\n\n {{range .results}}\n >ClusterID:{{index .group_values 0}}\n >NodeID:{{index .group_values 1}}\n >Priority:{{.priority}}\n >Link:[View Cluster Monitoring]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}) \n {{end}}\"\n }\n}\n" - } - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calavvp7h710dpnp32r3 -{ - "id": "builtin-calavvp7h710dpnp32r3", - "created": "2022-06-16T04:22:23.001354546Z", - "updated": "2022-07-21T23:10:36.70696738Z", - "name": "Index Health Change to Red", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]index" - ], - "filter": {}, - "raw_filter": { - "match_phrase": { - "metadata.labels.health_status": "red" - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.cluster_id", - "limit": 5 - }, - { - "field": "metadata.index_name", - "limit": 5 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "metadata.index_name", - "statistic": "count" - } - ], - "format_type": "num", - "expression": "count(metadata.index_name)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "1" - ], - "priority": "high" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "Health of Indices ({{.first_group_value}} ..., {{len .results}} indices in total) Changed to Red", - "message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex name:{{index .group_values 1}}; {{end}}", - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}?_g=%7B%22tab%22%3A%22indices%22%7D|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8 -{ - "id": "builtin-calaqnh7h710dpnp2bm8", - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-07-21T23:12:07.142532243Z", - "name": "JVM utilization is Too High", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]metrics*" - ], - "filter": {}, - "raw_filter": { - "bool": { - "must": [ - { - "term": { - "metadata.name": { - "value": "node_stats" - } - } - }, - { - "term": { - "metadata.category": { - "value": "elasticsearch" - } - } - } - ] - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.node_id", - "limit": 300 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.node_stats.jvm.mem.heap_used_percent", - "statistic": "p90" - } - ], - "format_type": "ratio", - "expression": "p90(payload.elasticsearch.node_stats.jvm.mem.heap_used_percent)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "80" - ], - "priority": "low" - }, - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "90" - ], - "priority": "medium" - }, - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "95" - ], - "priority": "high" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "JVM Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%", - "message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNode name:{{index .group_values 1}}; memory used percent：{{.result_value | to_fixed 2}}%;{{end}}", - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "3h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7 -{ - "id": "builtin-cbp2e4ianisjmu4giqs7", - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-08-09T09:39:29.604751601Z", - "name": "Search latency is great than 500ms", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]metrics*" - ], - "filter": {}, - "raw_filter": { - "bool": { - "must": [ - { - "term": { - "metadata.name": { - "value": "index_stats" - } - } - }, - { - "term": { - "metadata.category": { - "value": "elasticsearch" - } - } - } - ], - "must_not": [ - { - "term": { - "metadata.labels.index_name": { - "value": "_all" - } - } - } - ] - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.index_name", - "limit": 500 - } - ], - "formula": "a/b", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.index_stats.total.search.query_time_in_millis", - "statistic": "rate" - }, - { - "name": "b", - "field": "payload.elasticsearch.index_stats.primaries.search.query_total", - "statistic": "rate" - } - ], - "format_type": "num", - "expression": "rate(payload.elasticsearch.index_stats.total.search.query_time_in_millis)/rate(payload.elasticsearch.index_stats.primaries.search.query_total)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "500" - ], - "priority": "medium" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "Search latency is great than 500ms", - "message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex Name:{{index .group_values 1}}; \nCurrent Value:{{.result_value | to_fixed 2}}ms;\n{{end}}", - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Latency:* {{.result_value | to_fixed 2}}ms\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6 -{ - "id": "builtin-calgapp7h710dpnpbeb6", - "created": "2022-06-16T10:26:47.360988761Z", - "updated": "2022-07-22T00:03:34.044562893Z", - "name": "Shard Storage >= 55G", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]metrics*" - ], - "filter": {}, - "raw_filter": { - "range": { - "payload.elasticsearch.index_stats.shard_info.store_in_bytes": { - "gte": 59055800320 - } - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.index_name", - "limit": 500 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.index_stats.shard_info.store_in_bytes", - "statistic": "max" - } - ], - "format_type": "bytes", - "expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "53687091200" - ], - "priority": "high" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "Shard Storage >55GB in ({{.first_group_value}} ..., {{len .results}} indices in total)", - "message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }};\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22});\nMax Shard Storage：{{.result_value | format_bytes 2}};\n{{end}}", - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Max Shard Storage:* {{.result_value | format_bytes 2}}\"\n },\n \n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "24h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/doc/cb34sfl6psfiqtovhpt4 -{ - "id": "cb34sfl6psfiqtovhpt4", - "created": "2022-07-07T03:08:46.297166036Z", - "updated": "2022-08-09T08:40:05.323148338Z", - "name": "Too Many Deleted Documents", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]metrics*" - ], - "filter": {}, - "raw_filter": { - "range": { - "payload.elasticsearch.cluster_stats.indices.store.size_in_bytes": { - "gte": 32212254720 - } - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.index_name", - "limit": 300 - } - ], - "formula": "(a/(a+b))*100", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.index_stats.primaries.docs.deleted", - "statistic": "max" - }, - { - "name": "b", - "field": "payload.elasticsearch.index_stats.primaries.docs.count", - "statistic": "max" - } - ], - "format_type": "ratio", - "expression": "(max(payload.elasticsearch.index_stats.primaries.docs.deleted)/(max(payload.elasticsearch.index_stats.primaries.docs.deleted)+max(payload.elasticsearch.index_stats.primaries.docs.count)))*100" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "30" - ], - "priority": "medium" - }, - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "40" - ], - "priority": "high" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "Too Many Deleted Documents (>30%)", - "message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex:{{index .group_values 0}}; \nRatio of Deleted Documents:{{.result_value}};\n{{end}}", - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Deleted:* {{.result_value | to_fixed 2}}%\"\n },\n \n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "24h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]channnel/doc/builtin-cgnb2nt3q95nmusjl65g -{ - "id": "builtin-cgnb2nt3q95nmusjl65g", - "created": "2023-04-06T11:47:43.104108279Z", - "updated": "2023-08-04T10:34:29.112776+08:00", - "name": "Slack Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json", - "Content-type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"【Demo】Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"high\"}} \"#EB4C21\" {{else if eq .priority \"medium\"}} \"#FFB449\" {{else if eq .priority \"low\"}} \"#87d068\" {{else}} \"#2db7f5\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{index .group_values 0 | lookup \"category=metadata, object=cluster, property=name, default=N/A\"}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|View Cluster Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - }, - "sub_type": "slack" -} -POST $[[INDEX_PREFIX]]channnel/doc/builtin-cgiospt3q95q49k3u00g -{ - "id": "builtin-cgiospt3q95q49k3u00g", - "created": "2023-03-30T13:28:07.531263747Z", - "updated": "2023-08-04T11:13:51.608186+08:00", - "name": "DingTalk", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json", - "Content-type": "application/json" - }, - "method": "POST", - "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", - "body": "{\"msgtype\": \"text\",\"text\": {\"content\":\"------------------------------------\\n【 INFINI Platform Alerting 】\\n{{.title}}\\n------------------------------------\\n{{.message}}\\nLink:{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"}}" - }, - "sub_type": "dingtalk" -} -POST $[[INDEX_PREFIX]]channnel/doc/builtin-ch1os6t3q95lk6lepkq0 -{ - "id": "builtin-ch1os6t3q95lk6lepkq0", - "created": "2023-04-22T07:34:51.848540351Z", - "updated": "2023-08-04T10:34:13.937983+08:00", - "name": "Feishu Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json", - "Content-type": "application/json" - }, - "method": "POST", - "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msg_type\": \"text\",\n \"content\": \"{\\\"text\\\":\\\"Alerting: {{.title}} \\\\n Link:{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\\\"}\"\n}" - }, - "sub_type": "feishu" -} -POST $[[INDEX_PREFIX]]channnel/doc/builtin-cgnb2kt3q95nmusjl64g -{ - "id": "builtin-cgnb2kt3q95nmusjl64g", - "created": "2023-04-06T11:47:31.161587662Z", - "updated": "2023-08-04T10:33:54.594583+08:00", - "name": "Wechat Notification", + "id": "cj865st3q95rega919ig", + "created": "2023-08-07T11:20:19.223545026+08:00", + "updated": "2023-08-08T18:42:26.506499014+08:00", + "name": "[Alerting] Discord", "type": "webhook", "webhook": { "header_params": { "Content-Type": "application/json" }, "method": "POST", - "url": "{{$.env.WECHAT_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n{{.title}}\\n{{.message}}\"\n }\n}" + "url": "{{$.env.DISCORD_WEBHOOK_ENDPOINT}}", + "body": "{\n \"content\": \"Hello Alerting\"\n}" }, - "sub_type": "wechat" + "sub_type": "discord", + "enabled": false } -POST $[[INDEX_PREFIX]]channnel/doc/builtin-cgnb2r53q95nmusjl6vg +POST $[[INDEX_PREFIX]]channel/doc/cj86l0l3q95rrpfea6ug { - "id": "builtin-cgnb2r53q95nmusjl6vg", + "id": "cj86l0l3q95rrpfea6ug", + "created": "2023-08-07T11:52:34.192522006+08:00", + "updated": "2023-08-08T18:42:30.162079286+08:00", + "name": "[Recovery] Discord\t", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DISCORD_WEBHOOK_ENDPOINT}}", + "body": "{\n\n}" + }, + "sub_type": "discord", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cgnb2nt3q95nmusjl65g +{ + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-08T22:19:08.601341574+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing !*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.trigger_at | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Cluster:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cj8bq8d3q95ogankugqg +{ + "id": "cj8bq8d3q95ogankugqg", + "created": "2023-08-07T17:45:05.534408059+08:00", + "updated": "2023-08-08T19:26:34.009668892+08:00", + "name": "[Recovery] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*:rainbow: Alert [{{.rule_name}}] Resolved*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*ResolveAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Target:* {{.resource_name}}-{{.objects}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.trigger_at | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Duration:* {{.duration}}\"\n }\n },\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n }\n ]\n}" + }, + "sub_type": "slack", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cgiospt3q95q49k3u00g +{ + "id": "cgiospt3q95q49k3u00g", + "created": "2023-03-30T13:28:07.531263747Z", + "updated": "2023-08-08T22:19:07.545051029+08:00", + "name": "[Alerting] DingTalk Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.com/img/email/alert-header.png)\\n\\n🔥 **{{.title}}**\\n\\nIncident [{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing !\\n\\nPriority: {{.priority}}\\n\\nEventID: {{.event_id}}\\n\\nTarget: {{.resource_name}}-{{.objects}}\\n\\nTriggerAt: {{.trigger_at | datetime}}\\n\\n---\\n\\n{{.message}}\"\n }\n}" + }, + "sub_type": "dingtalk", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cgnb2r53q95nmusjl6vg +{ + "id": "cgnb2r53q95nmusjl6vg", "created": "2023-04-06T11:47:56.652637309Z", - "updated": "2023-08-04T10:12:44.675016+08:00", - "name": "SMS Notification", + "updated": "2023-08-08T19:49:20.312590885+08:00", + "name": "[Alerting] Email Notification", "type": "email", "sub_type": "email", "email": { "server_id": "", "recipients": { - "to": [] + "to": [], + "cc": [] }, - "subject": "{{.title}}", - "body": "Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\n{{.message}}" + "subject": "[INFINI Platform Alerting] 🔥 {{.title}}", + "body": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n
\n \"email-header\"\n
\n
\n \n \n \n \n \n \n
\n
\n
\n {{.title}}\n

\n \n

Priority: {{.priority}}

\n

EventID: {{.event_id}}

\n

Target: {{.resource_name}}-{{.objects}}

\n

TriggerAt: {{.trigger_at | datetime}}

\n {{.message | md_to_html}}\n
\n

\n \n \n View Detail\n \n

\n \n \n \n

\n \n

\n \"INFINI\n
\n
\n \n
\n

\n \n \n
\n
\n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n
\n
\n \n \n \n
\n
\n \n
\n \n \n
\n
\n \n ", + "content_type": "text/html" + }, + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/ch1os6t3q95lk6lepkq0 +{ + "id": "ch1os6t3q95lk6lepkq0", + "created": "2023-04-22T07:34:51.848540351Z", + "updated": "2023-08-09T09:29:26.412223281+08:00", + "name": "[Alerting] Feishu Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI Platform Alerting ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"{{if eq .priority \"critical\"}}red{{else if eq .priority \"high\"}}orange{{else if eq .priority \"medium\"}}yellow{{else if eq .priority \"low\"}}grey{{else}}blue{{end}}\"\n },\n \"elements\": [{\n \"tag\": \"markdown\",\n \"content\": \"🔥 Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing ! \\n **{{.title}}**\\nPriority: {{.priority}}\\nEventID: {{.event_id}}\\nTarget: {{.resource_name}}-{{.objects}}\\nTriggerAt: {{.trigger_at | datetime}}\"\n },{\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n }\n ]\n}\n}" + }, + "sub_type": "feishu", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cj8e9s53q95gsdbb054g +{ + "id": "cj8e9s53q95gsdbb054g", + "created": "2023-08-07T20:34:56.334695598+08:00", + "updated": "2023-08-08T21:34:50.261294305+08:00", + "name": "[Recovery] Feishu Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI Platform Alerting ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"green\"\n },\n \"elements\": [\n {\n \"tag\": \"markdown\",\n \"content\": \"🌈 **{{.title}}**\"\n },\n {\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n }\n ]\n }\n}" + }, + "sub_type": "feishu", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cj8ctat3q95l9ebbntlg + { + "id": "cj8ctat3q95l9ebbntlg", + "created": "2023-08-07T18:59:55.28732241+08:00", + "updated": "2023-08-08T19:46:30.557046793+08:00", + "name": "[Recovery] DingTalk Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.com/img/email/recovery-header.png)\\n\\n🌈 **{{.title}}**\\n\\n{{.message}}\\n\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n }\n}\n" + }, + "sub_type": "dingtalk", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cgnb2kt3q95nmusjl64g +{ + "id": "cgnb2kt3q95nmusjl64g", + "created": "2023-04-06T11:47:31.161587662Z", + "updated": "2023-08-08T22:19:06.712911427+08:00", + "name": "[Alerting] Wechat Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.WECOM_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"**[ INFINI Platform Alerting ]**\\n🔥 Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing !\\n**{{.title}}**\\nPriority: {{.priority}}\\n\\nEventID: {{.event_id}}\\n\\nTarget: {{.resource_name}}-{{.objects}}\\n\\nTriggerAt: {{.trigger_at | datetime}}\\n{{.message}}\"\n }\n}" + }, + "sub_type": "wechat", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cj8e9gt3q95gsdbb0170 +{ + "id": "cj8e9gt3q95gsdbb0170", + "created": "2023-08-07T20:34:11.998953512+08:00", + "updated": "2023-08-08T19:47:08.270014715+08:00", + "name": "[Recovery] Wechat Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.WECOM_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"**[ INFINI Platform Alerting ]**\\n🌈 **{{.title}}**\\n\\n{{.message}}\\n\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n }\n}\n" + }, + "sub_type": "wechat", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cj8atf53q95lhahebg8g +{ + "id": "cj8atf53q95lhahebg8g", + "created": "2023-08-07T16:43:40.062389175+08:00", + "updated": "2023-08-08T19:50:15.803258835+08:00", + "name": "[Recovery] Email Notification", + "type": "email", + "sub_type": "email", + "email": { + "server_id": "", + "recipients": { + "to": [], + "cc": [] + }, + "subject": "[INFINI Platform Alerting] 🌈 {{.title}}", + "body": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n
\n \"email-header\"\n
\n
\n \n \n \n \n \n \n
\n
\n
\n 🌈 {{.title}}\n

\n {{.message | md_to_html}}\n
\n

\n \n \n View Detail\n \n

\n \n \n \n

\n \n

\n \"INFINI\n
\n
\n \n
\n

\n \n \n
\n
\n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n
\n
\n \n \n \n
\n
\n \n
\n \n \n
\n
\n \n ", + "content_type": "text/html" + }, + "enabled": false +} + +#alerting +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6 +{ + "id": "builtin-calgapp7h710dpnpbeb6", + "created": "2022-06-16T10:26:47.360988761Z", + "updated": "2023-08-09T09:44:58.584645596+08:00", + "name": "Shard Storage >= 55G", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "range": { + "payload.elasticsearch.index_stats.shard_info.store_in_bytes": { + "gte": 59055800320 + } + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.index_name", + "limit": 500 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.index_stats.shard_info.store_in_bytes", + "statistic": "max" + } + ], + "format_type": "bytes", + "bucket_label": { + "enabled": false + }, + "expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "59055800320" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Shard Storage >55GB in ({{len .results}} indices in total)", + "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}), Max Shard Storage: {{.result_value | format_bytes 2}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T14:02:53.734855705+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Max shard storage: {{.result_value | format_bytes 2}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "- EventID: {{.event_id}}\n- Target: {{.resource_name}}-{{.objects}}\n- TriggerAt: {{.trigger_at}}\n- ResolveAt: {{.timestamp | datetime}}\n- Duration: {{.duration}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cb34sfl6psfiqtovhpt4 +{ + "id": "builtin-cb34sfl6psfiqtovhpt4", + "created": "2022-07-07T03:08:46.297166036Z", + "updated": "2023-08-09T09:45:34.123901475+08:00", + "name": "Too Many Deleted Documents", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "range": { + "payload.elasticsearch.cluster_stats.indices.store.size_in_bytes": { + "gte": 32212254720 + } + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 20 + }, + { + "field": "metadata.labels.index_name", + "limit": 10 + } + ], + "formula": "(a/(a+b))*100", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.index_stats.primaries.docs.deleted", + "statistic": "max" + }, + { + "name": "b", + "field": "payload.elasticsearch.index_stats.primaries.docs.count", + "statistic": "max" + } + ], + "format_type": "ratio", + "bucket_label": { + "enabled": false + }, + "expression": "(max(payload.elasticsearch.index_stats.primaries.docs.deleted)/(max(payload.elasticsearch.index_stats.primaries.docs.deleted)+max(payload.elasticsearch.index_stats.primaries.docs.count)))*100" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "30" + ], + "priority": "medium" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "40" + ], + "priority": "high" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "55" + ], + "priority": "low" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Too Many Deleted Documents (>30%)", + "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), Deleted: {{.result_value | to_fixed 2}}%\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "name": "", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Deleted ratio: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "24h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} + +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp20n2anisjmu4gehc5 +{ + "id": "builtin-cbp20n2anisjmu4gehc5", + "created": "2022-08-09T08:52:44.63345561Z", + "updated": "2023-08-09T09:43:37.945659792+08:00", + "name": "Elasticsearch node left cluster", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_node" + ], + "filter": {}, + "raw_filter": { + "match_phrase": { + "metadata.labels.status": "unavailable" + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.cluster_id", + "limit": 5 + }, + { + "field": "metadata.node_id", + "limit": 50 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "metadata.labels.status", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(metadata.labels.status)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "critical" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Elasticsearch node left cluster", + "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), Left: {{.result_value}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T10:42:17.686776304+08:00", + "name": "Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Left: {{.result_value}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calavvp7h710dpnp32r3 +{ + "id": "builtin-calavvp7h710dpnp32r3", + "created": "2022-06-16T04:22:23.001354546Z", + "updated": "2023-08-09T09:43:58.551403706+08:00", + "name": "Index Health Change to Red", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_index" + ], + "filter": {}, + "raw_filter": { + "match": { + "metadata.labels.health_status": "red" + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.cluster_id", + "limit": 50 + }, + { + "field": "metadata.index_name", + "limit": 1000 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "metadata.index_name", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(metadata.index_name)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Health of Indices ({{len .results}} indices in total) Changed to Red", + "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}) is Red now\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T15:17:26.18861218+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}> is Red now\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7 +{ + "id": "builtin-cbp2e4ianisjmu4giqs7", + "created": "2022-06-16T04:11:10.242061032Z", + "updated": "2023-08-09T09:44:31.495696286+08:00", + "name": "Search latency is great than 500ms", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "index_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ], + "must_not": [ + { + "term": { + "metadata.labels.index_name": { + "value": "_all" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 50 + }, + { + "field": "metadata.labels.index_name", + "limit": 10 + } + ], + "formula": "a/b", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.index_stats.total.search.query_time_in_millis", + "statistic": "rate" + }, + { + "name": "b", + "field": "payload.elasticsearch.index_stats.primaries.search.query_total", + "statistic": "rate" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "rate(payload.elasticsearch.index_stats.total.search.query_time_in_millis)/rate(payload.elasticsearch.index_stats.primaries.search.query_total)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "500" + ], + "priority": "medium" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "300" + ], + "priority": "low" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Search latency is great than 500ms", + "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}), Latency: {{.result_value | to_fixed 2}}ms\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-06T15:46:34.404507399+08:00", + "name": "Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "\n{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Search latency: {{.result_value | to_fixed 2}}ms\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8 +{ + "id": "builtin-calaqnh7h710dpnp2bm8", + "created": "2022-06-16T04:11:10.242061032Z", + "updated": "2023-08-09T09:46:34.428920151+08:00", + "name": "JVM utilization is Too High", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "node_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.node_id", + "limit": 300 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.node_stats.jvm.mem.heap_used_percent", + "statistic": "p90" + } + ], + "format_type": "ratio", + "bucket_label": { + "enabled": false + }, + "expression": "p90(payload.elasticsearch.node_stats.jvm.mem.heap_used_percent)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "80" + ], + "priority": "low" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "90" + ], + "priority": "medium" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "95" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "JVM Usage of Nodes ({{len .results}} nodes in total) >= {{.first_threshold}}%", + "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), JVM Usage: {{.result_value | to_fixed 2}}%\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-06T15:46:34.404507399+08:00", + "name": "Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, JVM Usage: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2 +{ + "id": "builtin-calakp97h710dpnp1fa2", + "created": "2022-06-16T03:58:29.437447113Z", + "updated": "2023-08-09T09:42:57.901272952+08:00", + "name": "CPU utilization is Too High", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "node_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.node_id", + "limit": 300 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.node_stats.process.cpu.percent", + "statistic": "avg" + } + ], + "format_type": "ratio", + "bucket_label": { + "enabled": false + }, + "expression": "avg(payload.elasticsearch.node_stats.process.cpu.percent)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "85" + ], + "priority": "low" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "90" + ], + "priority": "medium" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "95" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "CPU Usage of Nodes ({{len .results}} nodes in total) >= {{.first_threshold}}%", + "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), CPU Usage: {{.result_value | to_fixed 2}}%\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T15:17:26.18861218+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, CPU Usage: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "6h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnogps1 +{ + "id": "builtin-cal8n7p7h710dpnogps1", + "created": "2022-06-16T03:11:01.445958361Z", + "updated": "2023-08-09T09:43:16.31964237+08:00", + "name": "Disk utilization is Too High", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "node_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.node_id", + "limit": 200 + } + ], + "formula": "((a-b)/a)*100", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.node_stats.fs.data.total_in_bytes", + "statistic": "max" + }, + { + "name": "b", + "field": "payload.elasticsearch.node_stats.fs.data.free_in_bytes", + "statistic": "max" + } + ], + "format_type": "ratio", + "bucket_label": { + "enabled": false + }, + "expression": "((max(payload.elasticsearch.node_stats.fs.data.total_in_bytes)-max(payload.elasticsearch.node_stats.fs.data.free_in_bytes))/max(payload.elasticsearch.node_stats.fs.data.total_in_bytes))*100" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 5, + "operator": "gte", + "values": [ + "80" + ], + "priority": "low" + }, + { + "minimum_period_match": 5, + "operator": "gte", + "values": [ + "90" + ], + "priority": "medium" + }, + { + "minimum_period_match": 5, + "operator": "gte", + "values": [ + "95" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Disk Usage of Nodes ({{len .results}} nodes in total) >= {{.first_threshold}}%", + "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), Usage: {{.result_value | to_fixed 2}}% / Free: {{.relation_values.b | format_bytes 2}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "name": "", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Disk Usage: {{.result_value | to_fixed 2}}%, Free: {{.relation_values.b | format_bytes 2}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n },\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Document\" \n },\n \"style\": \"primary\",\n \"url\": \"https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-cluster.html#disk-based-shard-allocation\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "6h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0 +{ + "id": "builtin-cal8n7p7h710dpnoaps0", + "created": "2022-06-16T01:47:11.326727124Z", + "updated": "2023-08-09T09:50:05.833535441+08:00", + "name": "Cluster Health Change to Red", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "match": { + "payload.elasticsearch.cluster_health.status": "red" + } + }, + { + "term": { + "metadata.name": { + "value": "cluster_health" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.cluster_health.status", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(payload.elasticsearch.cluster_health.status)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "critical" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Health of Clusters ({{len .results}} clusters in total) Changed to Red", + "message": "{{range .results}}\nCluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/) is Red now\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T15:02:17.165625799+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Cluster:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}> is Red now\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" } } diff --git a/config/initialization_v6.tpl b/config/initialization_v6.tpl index 4012f8ef..458c4506 100644 --- a/config/initialization_v6.tpl +++ b/config/initialization_v6.tpl @@ -621,1155 +621,1767 @@ PUT $[[INDEX_PREFIX]]activities-00001 } } -#alerting -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2 -{ - "id": "builtin-calakp97h710dpnp1fa2", - "created": "2022-06-16T03:58:29.437447113Z", - "updated": "2022-07-21T23:12:51.111569117Z", - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "name": "CPU utilization is Too High", - "enabled": false, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]metrics*" - ], - "filter": {}, - "raw_filter": { - "bool": { - "must": [ - { - "term": { - "metadata.name": { - "value": "node_stats" - } - } - }, - { - "term": { - "metadata.category": { - "value": "elasticsearch" - } - } - } - ] - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.node_id", - "limit": 300 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.node_stats.process.cpu.percent", - "statistic": "avg" - } - ], - "format_type": "ratio", - "expression": "avg(payload.elasticsearch.node_stats.process.cpu.percent)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "80" - ], - "priority": "low" - }, - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "90" - ], - "priority": "medium" - }, - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "95" - ], - "priority": "high" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "CPU Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%", - "message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \nCPU:{{.result_value | to_fixed 2}}%;\n{{end}}", - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "6h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - +#alerting channel #The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0 +POST $[[INDEX_PREFIX]]channel/doc/cj865st3q95rega919ig { - "id": "builtin-cal8n7p7h710dpnoaps0", - "created": "2022-06-16T01:47:11.326727124Z", - "updated": "2022-07-13T04:00:06.181994982Z", - "name": "Cluster Health Change to Red", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]metrics*" - ], - "filter": {}, - "raw_filter": { - "bool": { - "must": [ - { - "match": { - "payload.elasticsearch.cluster_health.status": "red" - } - }, - { - "term": { - "metadata.name": { - "value": "cluster_health" - } - } - } - ] - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.cluster_health.status", - "statistic": "count" - } - ], - "format_type": "num", - "expression": "count(payload.elasticsearch.cluster_health.status)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "1" - ], - "priority": "critical" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "Health of Cluster[s] ({{.first_group_value}} ..., {{len .results}} clusters in total) Changed to Red", - "message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}, Name:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }} is RED now;\n{{end}}", - "normal": [ - { - "created": "2022-06-16T01:47:11.326727124Z", - "updated": "2022-06-16T01:47:11.326727124Z", - "name": "Slack webhook", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|View Cluster Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - }, - { - "created": "2022-06-16T01:47:11.326727124Z", - "updated": "2022-06-16T01:47:11.326727124Z", - "name": "DingTalk", - "type": "webhook", - "enabled": true, - "webhook": { - "header_params": { - "Content-type": "application/json" - }, - "method": "POST", - "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", - "body": "{\"msgtype\": \"text\",\"text\": {\"content\":\"Alerting: \\n{{.title}}\\n\\n{{.message}}\\nLink:{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"}}" - } - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnogps1 -{ - "id": "builtin-cal8n7p7h710dpnogps1", - "created": "2022-06-16T03:11:01.445958361Z", - "updated": "2022-07-22T00:06:26.498903821Z", - "name": "Disk utilization is Too High", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]metrics*" - ], - "filter": {}, - "raw_filter": { - "bool": { - "must": [ - { - "term": { - "metadata.name": { - "value": "node_stats" - } - } - }, - { - "term": { - "metadata.category": { - "value": "elasticsearch" - } - } - } - ] - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.node_id", - "limit": 200 - } - ], - "formula": "((a-b)/a)*100", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.node_stats.fs.data.total_in_bytes", - "statistic": "max" - }, - { - "name": "b", - "field": "payload.elasticsearch.node_stats.fs.data.free_in_bytes", - "statistic": "max" - } - ], - "format_type": "ratio", - "expression": "((max(payload.elasticsearch.node_stats.fs.data.total_in_bytes)-max(payload.elasticsearch.node_stats.fs.data.free_in_bytes))/max(payload.elasticsearch.node_stats.fs.data.total_in_bytes))*100" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 5, - "operator": "gte", - "values": [ - "85" - ], - "priority": "low" - }, - { - "minimum_period_match": 5, - "operator": "gte", - "values": [ - "90" - ], - "priority": "medium" - }, - { - "minimum_period_match": 5, - "operator": "gte", - "values": [ - "95" - ], - "priority": "high" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "Disk Utilization is Too High", - "message": "Severity:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID：{{index .group_values 0}} ;\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID：{{index .group_values 1}} ;\nDisk Usage:{{.result_value | to_fixed 2}}%；Free Storage:{{.relation_values.b | format_bytes 2}}；\n{{end}}", - "normal": [ - { - "created": "0001-01-01T00:00:00Z", - "updated": "0001-01-01T00:00:00Z", - "name": "Slack", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Free:* {{.relation_values.b | format_bytes 2}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "3h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp20n2anisjmu4gehc5 -{ - "id": "builtin-cbp20n2anisjmu4gehc5", - "created": "2022-08-09T08:52:44.63345561Z", - "updated": "2022-08-09T08:52:44.633455664Z", - "name": "Elasticsearch node left cluster", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]node" - ], - "filter": {}, - "raw_filter": { - "match_phrase": { - "metadata.labels.status": "unavailable" - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.cluster_id", - "limit": 5 - }, - { - "field": "metadata.node_id", - "limit": 50 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "metadata.labels.status", - "statistic": "count" - } - ], - "format_type": "num", - "expression": "count(metadata.labels.status)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "1" - ], - "priority": "critical" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "Elasticsearch node left cluster", - "message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNodeID:{{index .group_values 1}}; \n{{end}}", - "normal": [ - { - "created": "2022-08-09T08:52:44.63345561Z", - "updated": "2022-08-09T08:52:44.63345561Z", - "name": "Wechat", - "type": "webhook", - "sub_type": "wechat", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.WECHAT_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n{{.title}}\\n\n {{range .results}}\n >ClusterID:{{index .group_values 0}}\n >NodeID:{{index .group_values 1}}\n >Priority:{{.priority}}\n >Link:[View Cluster Monitoring]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}) \n {{end}}\"\n }\n}\n" - } - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calavvp7h710dpnp32r3 -{ - "id": "builtin-calavvp7h710dpnp32r3", - "created": "2022-06-16T04:22:23.001354546Z", - "updated": "2022-07-21T23:10:36.70696738Z", - "name": "Index Health Change to Red", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]index" - ], - "filter": {}, - "raw_filter": { - "match_phrase": { - "metadata.labels.health_status": "red" - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.cluster_id", - "limit": 5 - }, - { - "field": "metadata.index_name", - "limit": 5 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "metadata.index_name", - "statistic": "count" - } - ], - "format_type": "num", - "expression": "count(metadata.index_name)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "1" - ], - "priority": "high" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "Health of Indices ({{.first_group_value}} ..., {{len .results}} indices in total) Changed to Red", - "message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex name:{{index .group_values 1}}; {{end}}", - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}?_g=%7B%22tab%22%3A%22indices%22%7D|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8 -{ - "id": "builtin-calaqnh7h710dpnp2bm8", - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-07-21T23:12:07.142532243Z", - "name": "JVM utilization is Too High", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]metrics*" - ], - "filter": {}, - "raw_filter": { - "bool": { - "must": [ - { - "term": { - "metadata.name": { - "value": "node_stats" - } - } - }, - { - "term": { - "metadata.category": { - "value": "elasticsearch" - } - } - } - ] - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.node_id", - "limit": 300 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.node_stats.jvm.mem.heap_used_percent", - "statistic": "p90" - } - ], - "format_type": "ratio", - "expression": "p90(payload.elasticsearch.node_stats.jvm.mem.heap_used_percent)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "80" - ], - "priority": "low" - }, - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "90" - ], - "priority": "medium" - }, - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "95" - ], - "priority": "high" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "JVM Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%", - "message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nNode name:{{index .group_values 1}}; memory used percent：{{.result_value | to_fixed 2}}%;{{end}}", - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Severity:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "3h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7 -{ - "id": "builtin-cbp2e4ianisjmu4giqs7", - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-08-09T09:39:29.604751601Z", - "name": "Search latency is great than 500ms", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]metrics*" - ], - "filter": {}, - "raw_filter": { - "bool": { - "must": [ - { - "term": { - "metadata.name": { - "value": "index_stats" - } - } - }, - { - "term": { - "metadata.category": { - "value": "elasticsearch" - } - } - } - ], - "must_not": [ - { - "term": { - "metadata.labels.index_name": { - "value": "_all" - } - } - } - ] - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.index_name", - "limit": 500 - } - ], - "formula": "a/b", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.index_stats.total.search.query_time_in_millis", - "statistic": "rate" - }, - { - "name": "b", - "field": "payload.elasticsearch.index_stats.primaries.search.query_total", - "statistic": "rate" - } - ], - "format_type": "num", - "expression": "rate(payload.elasticsearch.index_stats.total.search.query_time_in_millis)/rate(payload.elasticsearch.index_stats.primaries.search.query_total)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "500" - ], - "priority": "medium" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "Search latency is great than 500ms", - "message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName: {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex Name:{{index .group_values 1}}; \nCurrent Value:{{.result_value | to_fixed 2}}ms;\n{{end}}", - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Latency:* {{.result_value | to_fixed 2}}ms\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6 -{ - "id": "builtin-calgapp7h710dpnpbeb6", - "created": "2022-06-16T10:26:47.360988761Z", - "updated": "2022-07-22T00:03:34.044562893Z", - "name": "Shard Storage >= 55G", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]metrics*" - ], - "filter": {}, - "raw_filter": { - "range": { - "payload.elasticsearch.index_stats.shard_info.store_in_bytes": { - "gte": 59055800320 - } - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.index_name", - "limit": 500 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.index_stats.shard_info.store_in_bytes", - "statistic": "max" - } - ], - "format_type": "bytes", - "expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "53687091200" - ], - "priority": "high" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "Shard Storage >55GB in ({{.first_group_value}} ..., {{len .results}} indices in total)", - "message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};\nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }};\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22});\nMax Shard Storage：{{.result_value | format_bytes 2}};\n{{end}}", - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Max Shard Storage:* {{.result_value | format_bytes 2}}\"\n },\n \n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "24h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]alert-rule/doc/cb34sfl6psfiqtovhpt4 -{ - "id": "cb34sfl6psfiqtovhpt4", - "created": "2022-07-07T03:08:46.297166036Z", - "updated": "2022-08-09T08:40:05.323148338Z", - "name": "Too Many Deleted Documents", - "enabled": false, - "creator": { - "name": "$[[USERNAME]]", - "id": "$[[USER_ID]]" - }, - "resource": { - "resource_id": "$[[RESOURCE_ID]]", - "resource_name": "$[[RESOURCE_NAME]]", - "type": "elasticsearch", - "objects": [ - "$[[INDEX_PREFIX]]metrics*" - ], - "filter": {}, - "raw_filter": { - "range": { - "payload.elasticsearch.cluster_stats.indices.store.size_in_bytes": { - "gte": 32212254720 - } - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.index_name", - "limit": 300 - } - ], - "formula": "(a/(a+b))*100", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.index_stats.primaries.docs.deleted", - "statistic": "max" - }, - { - "name": "b", - "field": "payload.elasticsearch.index_stats.primaries.docs.count", - "statistic": "max" - } - ], - "format_type": "ratio", - "expression": "(max(payload.elasticsearch.index_stats.primaries.docs.deleted)/(max(payload.elasticsearch.index_stats.primaries.docs.deleted)+max(payload.elasticsearch.index_stats.primaries.docs.count)))*100" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "30" - ], - "priority": "medium" - }, - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "40" - ], - "priority": "high" - } - ] - }, - "notification_config": { - "enabled": false, - "title": "Too Many Deleted Documents (>30%)", - "message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nClusterName:{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\nIndex:{{index .group_values 0}}; \nRatio of Deleted Documents:{{.result_value}};\n{{end}}", - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "sub_type": "slack", - "enabled": true, - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n{\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\"\n},\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Deleted:* {{.result_value | to_fixed 2}}%\"\n },\n \n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "24h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - -#The `id` value is consistent with the `_id` value -POST $[[INDEX_PREFIX]]channnel/doc/builtin-cgnb2nt3q95nmusjl65g -{ - "id": "builtin-cgnb2nt3q95nmusjl65g", - "created": "2023-04-06T11:47:43.104108279Z", - "updated": "2023-08-04T10:34:29.112776+08:00", - "name": "Slack Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json", - "Content-type": "application/json" - }, - "method": "POST", - "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"【Demo】Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"high\"}} \"#EB4C21\" {{else if eq .priority \"medium\"}} \"#FFB449\" {{else if eq .priority \"low\"}} \"#87d068\" {{else}} \"#2db7f5\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterName:* {{index .group_values 0 | lookup \"category=metadata, object=cluster, property=name, default=N/A\"}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|View Cluster Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - }, - "sub_type": "slack" -} -POST $[[INDEX_PREFIX]]channnel/doc/builtin-cgiospt3q95q49k3u00g -{ - "id": "builtin-cgiospt3q95q49k3u00g", - "created": "2023-03-30T13:28:07.531263747Z", - "updated": "2023-08-04T11:13:51.608186+08:00", - "name": "DingTalk", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json", - "Content-type": "application/json" - }, - "method": "POST", - "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", - "body": "{\"msgtype\": \"text\",\"text\": {\"content\":\"------------------------------------\\n【 INFINI Platform Alerting 】\\n{{.title}}\\n------------------------------------\\n{{.message}}\\nLink:{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"}}" - }, - "sub_type": "dingtalk" -} -POST $[[INDEX_PREFIX]]channnel/doc/builtin-ch1os6t3q95lk6lepkq0 -{ - "id": "builtin-ch1os6t3q95lk6lepkq0", - "created": "2023-04-22T07:34:51.848540351Z", - "updated": "2023-08-04T10:34:13.937983+08:00", - "name": "Feishu Notification", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json", - "Content-type": "application/json" - }, - "method": "POST", - "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msg_type\": \"text\",\n \"content\": \"{\\\"text\\\":\\\"Alerting: {{.title}} \\\\n Link:{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\\\"}\"\n}" - }, - "sub_type": "feishu" -} -POST $[[INDEX_PREFIX]]channnel/doc/builtin-cgnb2kt3q95nmusjl64g -{ - "id": "builtin-cgnb2kt3q95nmusjl64g", - "created": "2023-04-06T11:47:31.161587662Z", - "updated": "2023-08-04T10:33:54.594583+08:00", - "name": "Wechat Notification", + "id": "cj865st3q95rega919ig", + "created": "2023-08-07T11:20:19.223545026+08:00", + "updated": "2023-08-08T18:42:26.506499014+08:00", + "name": "[Alerting] Discord", "type": "webhook", "webhook": { "header_params": { "Content-Type": "application/json" }, "method": "POST", - "url": "{{$.env.WECHAT_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\\n{{.title}}\\n{{.message}}\"\n }\n}" + "url": "{{$.env.DISCORD_WEBHOOK_ENDPOINT}}", + "body": "{\n \"content\": \"Hello Alerting\"\n}" }, - "sub_type": "wechat" + "sub_type": "discord", + "enabled": false } -POST $[[INDEX_PREFIX]]channnel/doc/builtin-cgnb2r53q95nmusjl6vg +POST $[[INDEX_PREFIX]]channel/doc/cj86l0l3q95rrpfea6ug { - "id": "builtin-cgnb2r53q95nmusjl6vg", + "id": "cj86l0l3q95rrpfea6ug", + "created": "2023-08-07T11:52:34.192522006+08:00", + "updated": "2023-08-08T18:42:30.162079286+08:00", + "name": "[Recovery] Discord\t", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DISCORD_WEBHOOK_ENDPOINT}}", + "body": "{\n\n}" + }, + "sub_type": "discord", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cgnb2nt3q95nmusjl65g +{ + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-08T22:19:08.601341574+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing !*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.trigger_at | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Cluster:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cj8bq8d3q95ogankugqg +{ + "id": "cj8bq8d3q95ogankugqg", + "created": "2023-08-07T17:45:05.534408059+08:00", + "updated": "2023-08-08T19:26:34.009668892+08:00", + "name": "[Recovery] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*:rainbow: Alert [{{.rule_name}}] Resolved*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*ResolveAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Target:* {{.resource_name}}-{{.objects}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.trigger_at | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Duration:* {{.duration}}\"\n }\n },\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n }\n ]\n}" + }, + "sub_type": "slack", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cgiospt3q95q49k3u00g +{ + "id": "cgiospt3q95q49k3u00g", + "created": "2023-03-30T13:28:07.531263747Z", + "updated": "2023-08-08T22:19:07.545051029+08:00", + "name": "[Alerting] DingTalk Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.com/img/email/alert-header.png)\\n\\n🔥 **{{.title}}**\\n\\nIncident [{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing !\\n\\nPriority: {{.priority}}\\n\\nEventID: {{.event_id}}\\n\\nTarget: {{.resource_name}}-{{.objects}}\\n\\nTriggerAt: {{.trigger_at | datetime}}\\n\\n---\\n\\n{{.message}}\"\n }\n}" + }, + "sub_type": "dingtalk", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cgnb2r53q95nmusjl6vg +{ + "id": "cgnb2r53q95nmusjl6vg", "created": "2023-04-06T11:47:56.652637309Z", - "updated": "2023-08-04T10:12:44.675016+08:00", - "name": "SMS Notification", + "updated": "2023-08-08T19:49:20.312590885+08:00", + "name": "[Alerting] Email Notification", "type": "email", "sub_type": "email", "email": { "server_id": "", "recipients": { - "to": [] + "to": [], + "cc": [] }, - "subject": "{{.title}}", - "body": "Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing\n{{.message}}" + "subject": "[INFINI Platform Alerting] 🔥 {{.title}}", + "body": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n
\n \"email-header\"\n
\n
\n \n \n \n \n \n \n
\n
\n
\n {{.title}}\n

\n \n

Priority: {{.priority}}

\n

EventID: {{.event_id}}

\n

Target: {{.resource_name}}-{{.objects}}

\n

TriggerAt: {{.trigger_at | datetime}}

\n {{.message | md_to_html}}\n
\n

\n \n \n View Detail\n \n

\n \n \n \n

\n \n

\n \"INFINI\n
\n
\n \n
\n

\n \n \n
\n
\n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n
\n
\n \n \n \n
\n
\n \n
\n \n \n
\n
\n \n ", + "content_type": "text/html" + }, + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/ch1os6t3q95lk6lepkq0 +{ + "id": "ch1os6t3q95lk6lepkq0", + "created": "2023-04-22T07:34:51.848540351Z", + "updated": "2023-08-09T09:29:26.412223281+08:00", + "name": "[Alerting] Feishu Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI Platform Alerting ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"{{if eq .priority \"critical\"}}red{{else if eq .priority \"high\"}}orange{{else if eq .priority \"medium\"}}yellow{{else if eq .priority \"low\"}}grey{{else}}blue{{end}}\"\n },\n \"elements\": [{\n \"tag\": \"markdown\",\n \"content\": \"🔥 Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing ! \\n **{{.title}}**\\nPriority: {{.priority}}\\nEventID: {{.event_id}}\\nTarget: {{.resource_name}}-{{.objects}}\\nTriggerAt: {{.trigger_at | datetime}}\"\n },{\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n }\n ]\n}\n}" + }, + "sub_type": "feishu", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cj8e9s53q95gsdbb054g +{ + "id": "cj8e9s53q95gsdbb054g", + "created": "2023-08-07T20:34:56.334695598+08:00", + "updated": "2023-08-08T21:34:50.261294305+08:00", + "name": "[Recovery] Feishu Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI Platform Alerting ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"green\"\n },\n \"elements\": [\n {\n \"tag\": \"markdown\",\n \"content\": \"🌈 **{{.title}}**\"\n },\n {\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n }\n ]\n }\n}" + }, + "sub_type": "feishu", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cj8ctat3q95l9ebbntlg + { + "id": "cj8ctat3q95l9ebbntlg", + "created": "2023-08-07T18:59:55.28732241+08:00", + "updated": "2023-08-08T19:46:30.557046793+08:00", + "name": "[Recovery] DingTalk Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.com/img/email/recovery-header.png)\\n\\n🌈 **{{.title}}**\\n\\n{{.message}}\\n\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n }\n}\n" + }, + "sub_type": "dingtalk", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cgnb2kt3q95nmusjl64g +{ + "id": "cgnb2kt3q95nmusjl64g", + "created": "2023-04-06T11:47:31.161587662Z", + "updated": "2023-08-08T22:19:06.712911427+08:00", + "name": "[Alerting] Wechat Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.WECOM_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"**[ INFINI Platform Alerting ]**\\n🔥 Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}) is ongoing !\\n**{{.title}}**\\nPriority: {{.priority}}\\n\\nEventID: {{.event_id}}\\n\\nTarget: {{.resource_name}}-{{.objects}}\\n\\nTriggerAt: {{.trigger_at | datetime}}\\n{{.message}}\"\n }\n}" + }, + "sub_type": "wechat", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cj8e9gt3q95gsdbb0170 +{ + "id": "cj8e9gt3q95gsdbb0170", + "created": "2023-08-07T20:34:11.998953512+08:00", + "updated": "2023-08-08T19:47:08.270014715+08:00", + "name": "[Recovery] Wechat Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.WECOM_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"**[ INFINI Platform Alerting ]**\\n🌈 **{{.title}}**\\n\\n{{.message}}\\n\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}})\"\n }\n}\n" + }, + "sub_type": "wechat", + "enabled": false +} +POST $[[INDEX_PREFIX]]channel/doc/cj8atf53q95lhahebg8g +{ + "id": "cj8atf53q95lhahebg8g", + "created": "2023-08-07T16:43:40.062389175+08:00", + "updated": "2023-08-08T19:50:15.803258835+08:00", + "name": "[Recovery] Email Notification", + "type": "email", + "sub_type": "email", + "email": { + "server_id": "", + "recipients": { + "to": [], + "cc": [] + }, + "subject": "[INFINI Platform Alerting] 🌈 {{.title}}", + "body": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n
\n \"email-header\"\n
\n
\n \n \n \n \n \n \n
\n
\n
\n 🌈 {{.title}}\n

\n {{.message | md_to_html}}\n
\n

\n \n \n View Detail\n \n

\n \n \n \n

\n \n

\n \"INFINI\n
\n
\n \n
\n

\n \n \n
\n
\n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n
\n
\n \n \n \n
\n
\n \n
\n \n \n
\n
\n \n ", + "content_type": "text/html" + }, + "enabled": false +} + +#alerting +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calgapp7h710dpnpbeb6 +{ + "id": "builtin-calgapp7h710dpnpbeb6", + "created": "2022-06-16T10:26:47.360988761Z", + "updated": "2023-08-09T09:44:58.584645596+08:00", + "name": "Shard Storage >= 55G", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "range": { + "payload.elasticsearch.index_stats.shard_info.store_in_bytes": { + "gte": 59055800320 + } + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.index_name", + "limit": 500 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.index_stats.shard_info.store_in_bytes", + "statistic": "max" + } + ], + "format_type": "bytes", + "bucket_label": { + "enabled": false + }, + "expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "59055800320" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Shard Storage >55GB in ({{len .results}} indices in total)", + "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}), Max Shard Storage: {{.result_value | format_bytes 2}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T14:02:53.734855705+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Max shard storage: {{.result_value | format_bytes 2}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "- EventID: {{.event_id}}\n- Target: {{.resource_name}}-{{.objects}}\n- TriggerAt: {{.trigger_at}}\n- ResolveAt: {{.timestamp | datetime}}\n- Duration: {{.duration}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cb34sfl6psfiqtovhpt4 +{ + "id": "builtin-cb34sfl6psfiqtovhpt4", + "created": "2022-07-07T03:08:46.297166036Z", + "updated": "2023-08-09T09:45:34.123901475+08:00", + "name": "Too Many Deleted Documents", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "range": { + "payload.elasticsearch.cluster_stats.indices.store.size_in_bytes": { + "gte": 32212254720 + } + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 20 + }, + { + "field": "metadata.labels.index_name", + "limit": 10 + } + ], + "formula": "(a/(a+b))*100", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.index_stats.primaries.docs.deleted", + "statistic": "max" + }, + { + "name": "b", + "field": "payload.elasticsearch.index_stats.primaries.docs.count", + "statistic": "max" + } + ], + "format_type": "ratio", + "bucket_label": { + "enabled": false + }, + "expression": "(max(payload.elasticsearch.index_stats.primaries.docs.deleted)/(max(payload.elasticsearch.index_stats.primaries.docs.deleted)+max(payload.elasticsearch.index_stats.primaries.docs.count)))*100" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "30" + ], + "priority": "medium" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "40" + ], + "priority": "high" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "55" + ], + "priority": "low" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Too Many Deleted Documents (>30%)", + "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), Deleted: {{.result_value | to_fixed 2}}%\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "name": "", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Deleted ratio: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "24h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} + +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp20n2anisjmu4gehc5 +{ + "id": "builtin-cbp20n2anisjmu4gehc5", + "created": "2022-08-09T08:52:44.63345561Z", + "updated": "2023-08-09T09:43:37.945659792+08:00", + "name": "Elasticsearch node left cluster", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_node" + ], + "filter": {}, + "raw_filter": { + "match_phrase": { + "metadata.labels.status": "unavailable" + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.cluster_id", + "limit": 5 + }, + { + "field": "metadata.node_id", + "limit": 50 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "metadata.labels.status", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(metadata.labels.status)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "critical" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Elasticsearch node left cluster", + "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), Left: {{.result_value}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T10:42:17.686776304+08:00", + "name": "Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Left: {{.result_value}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calavvp7h710dpnp32r3 +{ + "id": "builtin-calavvp7h710dpnp32r3", + "created": "2022-06-16T04:22:23.001354546Z", + "updated": "2023-08-09T09:43:58.551403706+08:00", + "name": "Index Health Change to Red", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_index" + ], + "filter": {}, + "raw_filter": { + "match": { + "metadata.labels.health_status": "red" + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.cluster_id", + "limit": 50 + }, + { + "field": "metadata.index_name", + "limit": 1000 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "metadata.index_name", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(metadata.index_name)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Health of Indices ({{len .results}} indices in total) Changed to Red", + "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}) is Red now\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T15:17:26.18861218+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}> is Red now\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cbp2e4ianisjmu4giqs7 +{ + "id": "builtin-cbp2e4ianisjmu4giqs7", + "created": "2022-06-16T04:11:10.242061032Z", + "updated": "2023-08-09T09:44:31.495696286+08:00", + "name": "Search latency is great than 500ms", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "index_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ], + "must_not": [ + { + "term": { + "metadata.labels.index_name": { + "value": "_all" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 50 + }, + { + "field": "metadata.labels.index_name", + "limit": 10 + } + ], + "formula": "a/b", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.index_stats.total.search.query_time_in_millis", + "statistic": "rate" + }, + { + "name": "b", + "field": "payload.elasticsearch.index_stats.primaries.search.query_total", + "statistic": "rate" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "rate(payload.elasticsearch.index_stats.total.search.query_time_in_millis)/rate(payload.elasticsearch.index_stats.primaries.search.query_total)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "500" + ], + "priority": "medium" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "300" + ], + "priority": "low" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Search latency is great than 500ms", + "message": "{{range .results}}\nIndex: [{{index .group_values 1}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}), Latency: {{.result_value | to_fixed 2}}ms\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-06T15:46:34.404507399+08:00", + "name": "Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "\n{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Search latency: {{.result_value | to_fixed 2}}ms\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calaqnh7h710dpnp2bm8 +{ + "id": "builtin-calaqnh7h710dpnp2bm8", + "created": "2022-06-16T04:11:10.242061032Z", + "updated": "2023-08-09T09:46:34.428920151+08:00", + "name": "JVM utilization is Too High", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "node_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.node_id", + "limit": 300 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.node_stats.jvm.mem.heap_used_percent", + "statistic": "p90" + } + ], + "format_type": "ratio", + "bucket_label": { + "enabled": false + }, + "expression": "p90(payload.elasticsearch.node_stats.jvm.mem.heap_used_percent)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "80" + ], + "priority": "low" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "90" + ], + "priority": "medium" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "95" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "JVM Usage of Nodes ({{len .results}} nodes in total) >= {{.first_threshold}}%", + "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), JVM Usage: {{.result_value | to_fixed 2}}%\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-06T15:46:34.404507399+08:00", + "name": "Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, JVM Usage: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-calakp97h710dpnp1fa2 +{ + "id": "builtin-calakp97h710dpnp1fa2", + "created": "2022-06-16T03:58:29.437447113Z", + "updated": "2023-08-09T09:42:57.901272952+08:00", + "name": "CPU utilization is Too High", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "node_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.node_id", + "limit": 300 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.node_stats.process.cpu.percent", + "statistic": "avg" + } + ], + "format_type": "ratio", + "bucket_label": { + "enabled": false + }, + "expression": "avg(payload.elasticsearch.node_stats.process.cpu.percent)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "85" + ], + "priority": "low" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "90" + ], + "priority": "medium" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "95" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "CPU Usage of Nodes ({{len .results}} nodes in total) >= {{.first_threshold}}%", + "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), CPU Usage: {{.result_value | to_fixed 2}}%\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T15:17:26.18861218+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, CPU Usage: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "6h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnogps1 +{ + "id": "builtin-cal8n7p7h710dpnogps1", + "created": "2022-06-16T03:11:01.445958361Z", + "updated": "2023-08-09T09:43:16.31964237+08:00", + "name": "Disk utilization is Too High", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "node_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.node_id", + "limit": 200 + } + ], + "formula": "((a-b)/a)*100", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.node_stats.fs.data.total_in_bytes", + "statistic": "max" + }, + { + "name": "b", + "field": "payload.elasticsearch.node_stats.fs.data.free_in_bytes", + "statistic": "max" + } + ], + "format_type": "ratio", + "bucket_label": { + "enabled": false + }, + "expression": "((max(payload.elasticsearch.node_stats.fs.data.total_in_bytes)-max(payload.elasticsearch.node_stats.fs.data.free_in_bytes))/max(payload.elasticsearch.node_stats.fs.data.total_in_bytes))*100" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 5, + "operator": "gte", + "values": [ + "80" + ], + "priority": "low" + }, + { + "minimum_period_match": 5, + "operator": "gte", + "values": [ + "90" + ], + "priority": "medium" + }, + { + "minimum_period_match": 5, + "operator": "gte", + "values": [ + "95" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Disk Usage of Nodes ({{len .results}} nodes in total) >= {{.first_threshold}}%", + "message": "{{range .results}}\nNode: [{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22}) of Cluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/), Usage: {{.result_value | to_fixed 2}}% / Free: {{.relation_values.b | format_bytes 2}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "name": "", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Disk Usage: {{.result_value | to_fixed 2}}%, Free: {{.relation_values.b | format_bytes 2}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n },\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Document\" \n },\n \"style\": \"primary\",\n \"url\": \"https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-cluster.html#disk-based-shard-allocation\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "6h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + } +} +POST $[[INDEX_PREFIX]]alert-rule/doc/builtin-cal8n7p7h710dpnoaps0 +{ + "id": "builtin-cal8n7p7h710dpnoaps0", + "created": "2022-06-16T01:47:11.326727124Z", + "updated": "2023-08-09T09:50:05.833535441+08:00", + "name": "Cluster Health Change to Red", + "enabled": true, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "match": { + "payload.elasticsearch.cluster_health.status": "red" + } + }, + { + "term": { + "metadata.name": { + "value": "cluster_health" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.cluster_health.status", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(payload.elasticsearch.cluster_health.status)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "critical" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "Health of Clusters ({{len .results}} clusters in total) Changed to Red", + "message": "{{range .results}}\nCluster: [{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}/) is Red now\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T15:02:17.165625799+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Cluster:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}> is Red now\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/alert/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "recovery_notification_config": { + "enabled": true, + "title": "Alert [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "normal": [ + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" } }