diff --git a/initialization.dsl b/initialization.dsl deleted file mode 100644 index beafaed7..00000000 --- a/initialization.dsl +++ /dev/null @@ -1,1615 +0,0 @@ -PUT _template/.infini -{ - "order": 0, - "index_patterns": [ - ".infini_*" - ], - "settings": { - "index": { - "max_result_window": "10000000", - "mapping": { - "total_fields": { - "limit": "20000" - } - }, - "analysis": { - "analyzer": { - "suggest_text_search": { - "filter": [ - "word_delimiter" - ], - "tokenizer": "classic" - } - } - }, - "number_of_shards": "1" - } - }, - "mappings": { - "dynamic_templates": [ - { - "strings": { - "mapping": { - "ignore_above": 256, - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ] - }, - "aliases": {} -} - -PUT _ilm/policy/ilm_.infini_metrics-30days-retention -{ - "policy": { - "phases": { - "hot": { - "min_age": "0ms", - "actions": { - "rollover": { - "max_age": "30d", - "max_size": "50gb" - }, - "set_priority": { - "priority": 100 - } - } - }, - "delete": { - "min_age": "30d", - "actions": { - "delete": { - } - } - } - } - } -} - -PUT _template/.infini_metrics-rollover -{ - "order" : 100000, - "index_patterns" : [ - ".infini_metrics*" - ], - "settings" : { - "index" : { - "format" : "7", - "lifecycle" : { - "name" : "ilm_.infini_metrics-30days-retention", - "rollover_alias" : ".infini_metrics" - }, - "codec" : "best_compression", - "number_of_shards" : "1", - "translog.durability":"async" - } - }, - "mappings" : { - "dynamic_templates" : [ - { - "strings" : { - "mapping" : { - "ignore_above" : 256, - "type" : "keyword" - }, - "match_mapping_type" : "string" - } - } - ] - }, - "aliases" : { } - } - - -PUT .infini_metrics-00001 -{ - "settings": { - "index.lifecycle.rollover_alias":".infini_metrics" - , "refresh_interval": "5s" - }, - "aliases":{ - ".infini_metrics":{ - "is_write_index":true - } - }, - "mappings": { - "dynamic_templates": [ - { - "strings": { - "match_mapping_type": "string", - "mapping": { - "ignore_above": 256, - "type": "keyword" - } - } - } - ], - "properties": { - "timestamp": { - "type": "date" - } - } - } -} - -PUT _template/.infini_logs-rollover -{ - "order": 100000, - "index_patterns": [ - ".infini_logs*" - ], - "settings": { - "index": { - "format": "7", - "lifecycle": { - "name" : "ilm_.infini_metrics-30days-retention", - "rollover_alias" : ".infini_logs" - }, - "codec": "best_compression", - "number_of_shards": "1", - "translog": { - "durability": "async" - } - } - }, - "mappings": { - "dynamic_templates": [ - { - "strings": { - "mapping": { - "ignore_above": 256, - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "payload.message": { - "type": "text" - }, - "timestamp": { - "type": "date" - } - } - }, - "aliases": {} -} - -PUT .infini_logs-00001 -{ - "settings": { - "index.lifecycle.rollover_alias":".infini_logs" - , "refresh_interval": "5s" - }, - "aliases":{ - ".infini_logs":{ - "is_write_index":true - } - } -} - - -PUT _template/.infini_requests_logging-rollover -{ - "order": 100000, - "index_patterns": [ - ".infini_requests_logging*" - ], - "settings": { - "index": { - "format": "7", - "lifecycle": { - "name" : "ilm_.infini_metrics-30days-retention", - "rollover_alias" : ".infini_requests_logging" - }, - "codec": "best_compression", - "number_of_shards": "1", - "translog": { - "durability": "async" - } - } - }, - "mappings": { - "dynamic_templates": [ - { - "strings": { - "mapping": { - "ignore_above": 256, - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "request": { - "properties": { - "body": { - "type": "text" - } - } - }, - "response": { - "properties": { - "body": { - "type": "text" - } - } - }, - "timestamp": { - "type": "date" - } - } - }, - "aliases": {} -} - -PUT .infini_requests_logging-00001 -{ - "settings": { - "index.lifecycle.rollover_alias":".infini_requests_logging" - , "refresh_interval": "5s" - }, - "aliases":{ - ".infini_requests_logging":{ - "is_write_index":true - } - } -} - - -PUT _template/.infini_async_bulk_results-rollover -{ - "order": 100000, - "index_patterns": [ - ".infini_async_bulk_results*" - ], - "settings": { - "index": { - "format": "7", - "lifecycle": { - "name" : "ilm_.infini_metrics-30days-retention", - "rollover_alias" : ".infini_async_bulk_results" - }, - "codec": "best_compression", - "number_of_shards": "1", - "translog": { - "durability": "async" - } - } - }, - "mappings": { - "dynamic_templates": [ - { - "strings": { - "mapping": { - "ignore_above": 256, - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "request": { - "properties": { - "body": { - "type": "text" - } - } - }, - "response": { - "properties": { - "body": { - "type": "text" - } - } - }, - "timestamp": { - "type": "date" - } - } - }, - "aliases": {} -} - -PUT .infini_async_bulk_results-00001 -{ - "settings": { - "index.lifecycle.rollover_alias":".infini_async_bulk_results" - , "refresh_interval": "5s" - }, - "aliases":{ - ".infini_async_bulk_results":{ - "is_write_index":true - } - } -} - - -PUT _template/.infini_alert-history-rollover -{ - "order" : 100000, - "index_patterns" : [ - ".infini_alert-history*" - ], - "settings" : { - "index" : { - "format" : "7", - "lifecycle" : { - "name" : "ilm_.infini_metrics-30days-retention", - "rollover_alias" : ".infini_alert-history" - }, - "codec" : "best_compression", - "number_of_shards" : "1", - "translog.durability":"async" - } - }, - "mappings" : { - "dynamic_templates" : [ - { - "strings" : { - "mapping" : { - "ignore_above" : 256, - "type" : "keyword" - }, - "match_mapping_type" : "string" - } - } - ] - }, - "aliases" : { } - } - - -PUT .infini_alert-history-00001 -{ - "settings": { - "index.lifecycle.rollover_alias":".infini_alert-history" - , "refresh_interval": "5s" - }, - "aliases":{ - ".infini_alert-history":{ - "is_write_index":true - } - }, - "mappings": { - "properties" : { - "condition" : { - "properties" : { - "items" : { - "properties" : { - "expression" : { - "type" : "keyword", - "ignore_above" : 256 - }, - "minimum_period_match" : { - "type" : "long" - }, - "operator" : { - "type" : "keyword", - "ignore_above" : 256 - }, - "severity" : { - "type" : "keyword", - "ignore_above" : 256 - }, - "values" : { - "type" : "keyword", - "ignore_above" : 256 - } - } - }, - "operator" : { - "type" : "keyword", - "ignore_above" : 256 - } - } - }, - "condition_result" : { - "type" : "object", - "enabled" : false - }, - "context" : { - "type" : "keyword", - "copy_to" : [ - "search_text" - ] - }, - "created" : { - "type" : "date" - }, - "expression" : { - "type" : "keyword", - "copy_to" : [ - "search_text" - ] - }, - "id" : { - "type" : "keyword" - }, - "is_escalated" : { - "type" : "boolean" - }, - "is_notified" : { - "type" : "boolean" - }, - "message" : { - "type" : "keyword", - "ignore_above" : 256 - }, - "objects" : { - "type" : "keyword", - "copy_to" : [ - "search_text" - ] - }, - "resource_id" : { - "type" : "keyword" - }, - "resource_name" : { - "type" : "keyword" - }, - "rule_id" : { - "type" : "keyword" - }, - "rule_name" : { - "type" : "keyword" - }, - "search_text" : { - "type" : "text", - "analyzer" : "suggest_text_search", - "index_prefixes" : { - "min_chars" : 2, - "max_chars" : 5 - }, - "index_phrases" : true - }, - "severity" : { - "type" : "keyword" - }, - "state" : { - "type" : "keyword", - "ignore_above" : 256 - }, - "title" : { - "type" : "keyword" - }, - "updated" : { - "type" : "date" - } - } - } -} - - -PUT _template/.infini_activities-rollover -{ - "order" : 100000, - "index_patterns" : [ - ".infini_activities*" - ], - "settings" : { - "index" : { - "format" : "7", - "lifecycle" : { - "name" : "ilm_.infini_metrics-30days-retention", - "rollover_alias" : ".infini_activities" - }, - "codec" : "best_compression", - "number_of_shards" : "1", - "translog.durability":"async" - } - }, - "mappings" : { - "dynamic_templates" : [ - { - "strings" : { - "mapping" : { - "ignore_above" : 256, - "type" : "keyword" - }, - "match_mapping_type" : "string" - } - } - ] - }, - "aliases" : { } - } - - -PUT .infini_activities-00001 -{ - "mappings": { - "dynamic_templates": [ - { - "strings": { - "match_mapping_type": "string", - "mapping": { - "ignore_above": 256, - "type": "keyword" - } - } - } - ], - "properties": { - "changelog": { - "type": "object", - "enabled": false - }, - "id": { - "type": "keyword" - }, - "metadata": { - "properties": { - "category": { - "type": "keyword", - "ignore_above": 256 - }, - "group": { - "type": "keyword", - "ignore_above": 256 - }, - "name": { - "type": "keyword", - "ignore_above": 256 - }, - "type": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "payload": { - "type": "object", - "enabled": false - }, - "timestamp": { - "type": "date" - } - } - }, - "settings": { - "index": { - "lifecycle.rollover_alias": ".infini_activities", - "refresh_interval": "5s", - "mapping": { - "total_fields": { - "limit": "20000" - } - }, - "max_result_window": "10000000", - "analysis": { - "analyzer": { - "suggest_text_search": { - "filter": [ - "word_delimiter" - ], - "tokenizer": "classic" - } - } - } - } - }, - "aliases": { - ".infini_activities": { - "is_write_index": true - } - } -} - - -#alerting -POST .infini_alert-rule/_doc/builtin-calakp97h710dpnp1fa2 -{ - "id": "builtin-calakp97h710dpnp1fa2", - "created": "2022-06-16T03:58:29.437447113Z", - "updated": "2022-07-21T23:12:51.111569117Z", - "name": "CPU utilization is Too High", - "enabled": false, - "resource": { - "resource_id": "infini_default_system_cluster", - "resource_name": "INFINI_SYSTEM (Shockwave)", - "type": "elasticsearch", - "objects": [ - ".infini_metrics*" - ], - "filter": {}, - "raw_filter": { - "bool": { - "must": [ - { - "term": { - "metadata.name": { - "value": "node_stats" - } - } - }, - { - "term": { - "metadata.category": { - "value": "elasticsearch" - } - } - } - ] - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.node_id", - "limit": 300 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.node_stats.process.cpu.percent", - "statistic": "avg" - } - ], - "format_type": "ratio", - "expression": "avg(payload.elasticsearch.node_stats.process.cpu.percent)", - "title": "CPU Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%", - "message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};NodeID:{{index .group_values 1}}; CPU:{{.result_value | to_fixed 2}}%;\n{{end}}" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "80" - ], - "priority": "low" - }, - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "90" - ], - "priority": "medium" - }, - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "95" - ], - "priority": "high" - } - ] - }, - "channels": { - "enabled": true, - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "${SLACK_WEBHOOK_ENDPOINT}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "6h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - -#The `id` value is consistent with the `_id` value -POST .infini_alert-rule/_doc/builtin-cal8n7p7h710dpnoaps0 -{ - "id": "builtin-cal8n7p7h710dpnoaps0", - "created": "2022-06-16T01:47:11.326727124Z", - "updated": "2022-07-13T04:00:06.181994982Z", - "name": "Cluster Health Change to Red", - "enabled": false, - "resource": { - "resource_id": "infini_default_system_cluster", - "resource_name": "INFINI_SYSTEM (Shockwave)", - "type": "elasticsearch", - "objects": [ - ".infini_metrics*" - ], - "filter": {}, - "raw_filter": { - "bool": { - "must": [ - { - "match": { - "payload.elasticsearch.cluster_health.status": "red" - } - }, - { - "term": { - "metadata.name": { - "value": "cluster_health" - } - } - } - ] - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.cluster_health.status", - "statistic": "count" - } - ], - "format_type": "num", - "expression": "count(payload.elasticsearch.cluster_health.status)", - "title": "Health of Cluster[s] ({{.first_group_value}} ..., {{len .results}} clusters in total) Changed to Red", - "message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}} is red now;\n{{end}}" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "1" - ], - "priority": "critical" - } - ] - }, - "channels": { - "enabled": true, - "normal": [ - { - "created": "2022-06-16T01:47:11.326727124Z", - "updated": "2022-06-16T01:47:11.326727124Z", - "name": "Slack webhook", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "${SLACK_WEBHOOK_ENDPOINT}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"high\"}} \"#EB4C21\" {{else if eq .priority \"medium\"}} \"#FFB449\" {{else if eq .priority \"low\"}} \"#87d068\" {{else}} \"#2db7f5\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|View Cluster Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - }, - { - "created": "2022-06-16T01:47:11.326727124Z", - "updated": "2022-06-16T01:47:11.326727124Z", - "name": "DingTalk", - "type": "webhook", - "webhook": { - "header_params": { - "Content-type": "application/json" - }, - "method": "POST", - "url": "${DINGTALK_WEBHOOK_ENDPOINT}", - "body": "{\"msgtype\": \"text\",\"text\": {\"content\":\"Alerting: \\n{{.title}}\\n\\n{{.message}}\\nLink:${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}\"}}" - } - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - - -#The `id` value is consistent with the `_id` value -POST .infini_alert-rule/_doc/builtin-cal8n7p7h710dpnogps1 -{ - "id": "builtin-cal8n7p7h710dpnogps1", - "created": "2022-06-16T03:11:01.445958361Z", - "updated": "2022-07-22T00:06:26.498903821Z", - "name": "Disk utilization is Too High", - "enabled": false, - "resource": { - "resource_id": "infini_default_system_cluster", - "resource_name": "INFINI_SYSTEM (Shockwave)", - "type": "elasticsearch", - "objects": [ - ".infini_metrics*" - ], - "filter": {}, - "raw_filter": { - "bool": { - "must": [ - { - "term": { - "metadata.name": { - "value": "node_stats" - } - } - }, - { - "term": { - "metadata.category": { - "value": "elasticsearch" - } - } - } - ] - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.node_id", - "limit": 200 - } - ], - "formula": "((a-b)/a)*100", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.node_stats.fs.data.total_in_bytes", - "statistic": "max" - }, - { - "name": "b", - "field": "payload.elasticsearch.node_stats.fs.data.free_in_bytes", - "statistic": "max" - } - ], - "format_type": "ratio", - "expression": "((max(payload.elasticsearch.node_stats.fs.data.total_in_bytes)-max(payload.elasticsearch.node_stats.fs.data.free_in_bytes))/max(payload.elasticsearch.node_stats.fs.data.total_in_bytes))*100", - "title": "Disk Utilization is Too High", - "message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}} ;\nNodeID:{{index .group_values 1}} ;\nDisk Usage:{{.result_value | to_fixed 2}}%;Free Storage:{{.relation_values.b | format_bytes 2}};\n{{end}}" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 5, - "operator": "gte", - "values": [ - "85" - ], - "priority": "low" - }, - { - "minimum_period_match": 5, - "operator": "gte", - "values": [ - "90" - ], - "priority": "medium" - }, - { - "minimum_period_match": 5, - "operator": "gte", - "values": [ - "95" - ], - "priority": "high" - } - ] - }, - "channels": { - "enabled": true, - "normal": [ - { - "created": "0001-01-01T00:00:00Z", - "updated": "0001-01-01T00:00:00Z", - "name": "Slack", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "${SLACK_WEBHOOK_ENDPOINT}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Free:* {{.relation_values.b | format_bytes 2}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "3h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - -#The `id` value is consistent with the `_id` value -POST .infini_alert-rule/_doc/builtin-cbp20n2anisjmu4gehc5 -{ - "id": "builtin-cbp20n2anisjmu4gehc5", - "created": "2022-08-09T08:52:44.63345561Z", - "updated": "2022-08-09T08:52:44.633455664Z", - "name": "Elasticsearch node left cluster", - "enabled": false, - "resource": { - "resource_id": "infini_default_system_cluster", - "resource_name": "INFINI_SYSTEM (Shockwave)", - "type": "elasticsearch", - "objects": [ - ".infini_node" - ], - "filter": {}, - "raw_filter": { - "match_phrase": { - "metadata.labels.status": "unavailable" - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.cluster_id", - "limit": 5 - }, - { - "field": "metadata.node_id", - "limit": 50 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "metadata.labels.status", - "statistic": "count" - } - ], - "format_type": "num", - "expression": "count(metadata.labels.status)", - "title": "Elasticsearch node left cluster", - "message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nNodeID:{{index .group_values 1}}; \n{{end}}" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "1" - ], - "priority": "critical" - } - ] - }, - "channels": { - "enabled": true, - "normal": [ - { - "created": "2022-08-09T08:52:44.63345561Z", - "updated": "2022-08-09T08:52:44.63345561Z", - "name": "Wechat", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "${WECHAT_WEBHOOK_ENDPOINT}", - "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"Incident [#{{.event_id}}](${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}) is ongoing\\n{{.title}}\\n\n {{range .results}}\n >ClusterID:{{index .group_values 0}}\n >NodeID:{{index .group_values 1}}\n >Priority:{{.priority}}\n >Link:[View Cluster Monitoring](${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}) \n {{end}}\"\n }\n}\n" - } - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - - -#The `id` value is consistent with the `_id` value -POST .infini_alert-rule/_doc/builtin-calavvp7h710dpnp32r3 -{ - "id": "builtin-calavvp7h710dpnp32r3", - "created": "2022-06-16T04:22:23.001354546Z", - "updated": "2022-07-21T23:10:36.70696738Z", - "name": "Index Health Change to Red", - "enabled": false, - "resource": { - "resource_id": "infini_default_system_cluster", - "resource_name": "INFINI_SYSTEM (Shockwave)", - "type": "elasticsearch", - "objects": [ - ".infini_index" - ], - "filter": {}, - "raw_filter": { - "match_phrase": { - "metadata.labels.health_status": "red" - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.cluster_id", - "limit": 5 - }, - { - "field": "metadata.index_name", - "limit": 5 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "metadata.index_name", - "statistic": "count" - } - ], - "format_type": "num", - "expression": "count(metadata.index_name)", - "title": "Health of Indices ({{.first_group_value}} ..., {{len .results}} indices in total) Changed to Red", - "message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; Index name:{{index .group_values 1}}; {{end}}" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "1" - ], - "priority": "high" - } - ] - }, - "channels": { - "enabled": true, - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "${SLACK_WEBHOOK_ENDPOINT}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}?_g=%7B%22tab%22%3A%22indices%22%7D|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - - -#The `id` value is consistent with the `_id` value -POST .infini_alert-rule/_doc/builtin-calaqnh7h710dpnp2bm8 -{ - "id": "builtin-calaqnh7h710dpnp2bm8", - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-07-21T23:12:07.142532243Z", - "name": "JVM utilization is Too High", - "enabled": false, - "resource": { - "resource_id": "infini_default_system_cluster", - "resource_name": "INFINI_SYSTEM (Shockwave)", - "type": "elasticsearch", - "objects": [ - ".infini_metrics*" - ], - "filter": {}, - "raw_filter": { - "bool": { - "must": [ - { - "term": { - "metadata.name": { - "value": "node_stats" - } - } - }, - { - "term": { - "metadata.category": { - "value": "elasticsearch" - } - } - } - ] - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.node_id", - "limit": 300 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.node_stats.jvm.mem.heap_used_percent", - "statistic": "p90" - } - ], - "format_type": "ratio", - "expression": "p90(payload.elasticsearch.node_stats.jvm.mem.heap_used_percent)", - "title": "JVM Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%", - "message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; Node name:{{index .group_values 1}}; memory used percent:{{.result_value | to_fixed 2}}%;{{end}}" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "80" - ], - "priority": "low" - }, - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "90" - ], - "priority": "medium" - }, - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "95" - ], - "priority": "high" - } - ] - }, - "channels": { - "enabled": true, - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "${SLACK_WEBHOOK_ENDPOINT}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "3h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - -#The `id` value is consistent with the `_id` value -POST .infini_alert-rule/_doc/builtin-cbp2e4ianisjmu4giqs7 -{ - "id": "builtin-cbp2e4ianisjmu4giqs7", - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-08-09T09:39:29.604751601Z", - "name": "Search latency is great than 500ms", - "enabled": false, - "resource": { - "resource_id": "infini_default_system_cluster", - "resource_name": "INFINI_SYSTEM (Shockwave)", - "type": "elasticsearch", - "objects": [ - ".infini_metrics*" - ], - "filter": {}, - "raw_filter": { - "bool": { - "must": [ - { - "term": { - "metadata.name": { - "value": "index_stats" - } - } - }, - { - "term": { - "metadata.category": { - "value": "elasticsearch" - } - } - } - ], - "must_not": [ - { - "term": { - "metadata.labels.index_name": { - "value": "_all" - } - } - } - ] - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.index_name", - "limit": 500 - } - ], - "formula": "a/b", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.index_stats.total.search.query_time_in_millis", - "statistic": "rate" - }, - { - "name": "b", - "field": "payload.elasticsearch.index_stats.primaries.search.query_total", - "statistic": "rate" - } - ], - "format_type": "num", - "expression": "rate(payload.elasticsearch.index_stats.total.search.query_time_in_millis)/rate(payload.elasticsearch.index_stats.primaries.search.query_total)", - "title": "Search latency is great than 500ms", - "message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nIndex name:{{index .group_values 1}}; \nCurrent value:{{.result_value | to_fixed 2}}ms;\n{{end}}" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "500" - ], - "priority": "medium" - } - ] - }, - "channels": { - "enabled": true, - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "${SLACK_WEBHOOK_ENDPOINT}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Latency:* {{.result_value | to_fixed 2}}ms\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "1h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - -#The `id` value is consistent with the `_id` value -POST .infini_alert-rule/_doc/builtin-calgapp7h710dpnpbeb6 -{ - "id": "builtin-calgapp7h710dpnpbeb6", - "created": "2022-06-16T10:26:47.360988761Z", - "updated": "2022-07-22T00:03:34.044562893Z", - "name": "Shard Storage >= 55G", - "enabled": false, - "resource": { - "resource_id": "infini_default_system_cluster", - "resource_name": "INFINI_SYSTEM (Shockwave)", - "type": "elasticsearch", - "objects": [ - ".infini_metrics*" - ], - "filter": {}, - "raw_filter": { - "range": { - "payload.elasticsearch.index_stats.shard_info.store_in_bytes": { - "gte": 59055800320 - } - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.index_name", - "limit": 500 - } - ], - "formula": "a", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.index_stats.shard_info.store_in_bytes", - "statistic": "max" - } - ], - "format_type": "bytes", - "expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)", - "title": "Shard Storage >55GB in ({{.first_group_value}} ..., {{len .results}} indices in total)", - "message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; Index:{{index .group_values 1}}; Max Shard Storage:{{.result_value | format_bytes 2}};{{end}}" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "53687091200" - ], - "priority": "high" - } - ] - }, - "channels": { - "enabled": true, - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "${SLACK_WEBHOOK_ENDPOINT}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Max Shard Storage:* {{.result_value | format_bytes 2}}\"\n },\n \n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "24h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - -#The `id` value is consistent with the `_id` value -POST .infini_alert-rule/_doc/cb34sfl6psfiqtovhpt4 -{ - "id": "cb34sfl6psfiqtovhpt4", - "created": "2022-07-07T03:08:46.297166036Z", - "updated": "2022-08-09T08:40:05.323148338Z", - "name": "Too Many Deleted Documents", - "enabled": false, - "resource": { - "resource_id": "infini_default_system_cluster", - "resource_name": "INFINI_SYSTEM (Shockwave)", - "type": "elasticsearch", - "objects": [ - ".infini_metrics*" - ], - "filter": {}, - "raw_filter": { - "range": { - "payload.elasticsearch.cluster_stats.indices.store.size_in_bytes": { - "gte": 32212254720 - } - } - }, - "time_field": "timestamp", - "context": { - "fields": null - } - }, - "metrics": { - "bucket_size": "1m", - "groups": [ - { - "field": "metadata.labels.cluster_id", - "limit": 5 - }, - { - "field": "metadata.labels.index_name", - "limit": 300 - } - ], - "formula": "(a/(a+b))*100", - "items": [ - { - "name": "a", - "field": "payload.elasticsearch.index_stats.primaries.docs.deleted", - "statistic": "max" - }, - { - "name": "b", - "field": "payload.elasticsearch.index_stats.primaries.docs.count", - "statistic": "max" - } - ], - "format_type": "ratio", - "expression": "(max(payload.elasticsearch.index_stats.primaries.docs.deleted)/(max(payload.elasticsearch.index_stats.primaries.docs.deleted)+max(payload.elasticsearch.index_stats.primaries.docs.count)))*100", - "title": "Too Many Deleted Documents (>30%)", - "message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nIndex:{{index .group_values 0}}; \nRatio of Deleted Documents:{{.result_value}};\n{{end}}" - }, - "conditions": { - "operator": "any", - "items": [ - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "30" - ], - "priority": "medium" - }, - { - "minimum_period_match": 1, - "operator": "gte", - "values": [ - "40" - ], - "priority": "high" - } - ] - }, - "channels": { - "enabled": true, - "normal": [ - { - "created": "2022-06-16T04:11:10.242061032Z", - "updated": "2022-06-16T04:11:10.242061032Z", - "name": "Slack", - "type": "webhook", - "webhook": { - "header_params": { - "Content-Type": "application/json" - }, - "method": "POST", - "url": "${SLACK_WEBHOOK_ENDPOINT}", - "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Deleted:* {{.result_value | to_fixed 2}}%\"\n },\n \n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" - } - } - ], - "throttle_period": "24h", - "accept_time_range": { - "start": "00:00", - "end": "23:59" - } - }, - "schedule": { - "interval": "1m" - } -} - - -GET / - -