diff --git a/Jenkinsfile b/Jenkinsfile index 37618649..9e96317e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,7 +32,7 @@ pipeline { sh label: 'copy-license', script: 'cd /home/jenkins/go/src/infini.sh/console && cp ../framework/LICENSE bin && cat ../framework/NOTICE NOTICE > bin/NOTICE' - sh label: 'copy-configs', script: 'cd /home/jenkins/go/src/infini.sh/console && mkdir -p bin/config && cp config/*.json bin/config' + sh label: 'copy-configs', script: 'cd /home/jenkins/go/src/infini.sh/console && mkdir -p bin/config && cp config/*.json bin/config && cp config/*.yml bin/config && cp config/*.tpl bin/config' sh label: 'package-linux-amd64', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-amd64.tar.gz console-linux-amd64 console.yml LICENSE NOTICE config' sh label: 'package-linux-386', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-386.tar.gz console-linux-386 console.yml LICENSE NOTICE config' diff --git a/env_check.go b/bootstrap_check.go similarity index 74% rename from env_check.go rename to bootstrap_check.go index 395c7df1..c934f070 100644 --- a/env_check.go +++ b/bootstrap_check.go @@ -8,14 +8,14 @@ import ( "fmt" "github.com/buger/jsonparser" log "github.com/cihub/seelog" - "infini.sh/console/config" "infini.sh/framework/core/elastic" "infini.sh/framework/core/env" + "infini.sh/framework/core/global" "infini.sh/framework/core/util" ) func bootstrapRequirementCheck() error{ - err := checkElasticsearchRequire() + err := checkElasticsearchRequirements() if err != nil { return err } @@ -23,7 +23,7 @@ func bootstrapRequirementCheck() error{ } -func checkElasticsearchRequire() error{ +func checkElasticsearchRequirements() error{ log.Trace("start to check elasticsearch requirement") var esConfigs = []elastic.ElasticsearchConfig{} ok, err := env.ParseConfig("elasticsearch", &esConfigs) @@ -33,27 +33,24 @@ func checkElasticsearchRequire() error{ if !ok { return fmt.Errorf("elasticsearch config section not found") } - appConfig = &config.AppConfig{ - Elasticsearch: "default", - } - ok, err = env.ParseConfig("web", appConfig) - if err != nil { - return fmt.Errorf("parse web config section error: %v", err) - } - if !ok { - return fmt.Errorf("web config section not found") - } - if appConfig.Elasticsearch == "" { - return fmt.Errorf("elasticsearch config of web section can not be empty") + + elasticsearchID:=global.Lookup(elastic.GlobalSystemElasticsearchID) + + if elasticsearchID == nil||elasticsearchID=="" { + return fmt.Errorf("elasticsearch config in web section can not be empty") } + + esID:=elasticsearchID.(string) + var targetEsConfig *elastic.ElasticsearchConfig for _, esConfig := range esConfigs { - if esConfig.Name == appConfig.Elasticsearch { + if esConfig.ID == esID||(esConfig.ID==""&&esConfig.Name==esID) { targetEsConfig = &esConfig } } + if targetEsConfig == nil { - return fmt.Errorf("elasticsearch config named %s not found", appConfig.Elasticsearch) + return fmt.Errorf("elasticsearch config %s was not found", esID) } var req = util.NewGetRequest(targetEsConfig.Endpoint, nil) if targetEsConfig.BasicAuth != nil { @@ -65,6 +62,10 @@ func checkElasticsearchRequire() error{ return fmt.Errorf("check elasticsearch requirement error: %v", err) } + if result==nil||result.Body==nil||len(result.Body)==0{ + return fmt.Errorf("failed to retrive elasticsearch version info") + } + versionNumber, err := jsonparser.GetString(result.Body, "version", "number") if err != nil { return fmt.Errorf("check elasticsearch requirement error: %v, got response: %s", err, string(result.Body)) diff --git a/config/config.go b/config/config.go index 94d1e6d8..a3cfb9a2 100644 --- a/config/config.go +++ b/config/config.go @@ -3,7 +3,6 @@ package config import "infini.sh/framework/core/config" type AppConfig struct { - Elasticsearch string `config:"elasticsearch"` UI UIConfig `config:"ui"` Network config.NetworkConfig `config:"network"` TLSConfig config.TLSConfig `config:"tls"` diff --git a/config/initialization.tpl b/config/initialization.tpl new file mode 100644 index 00000000..29332649 --- /dev/null +++ b/config/initialization.tpl @@ -0,0 +1,1475 @@ +PUT _template/$[[TEMPLATE_NAME]] +{ + "order": 0, + "index_patterns": [ + "$[[INDEX_PREFIX]]*" + ], + "settings": { + "index": { + "max_result_window": "10000000", + "mapping": { + "total_fields": { + "limit": "20000" + } + }, + "analysis": { + "analyzer": { + "suggest_text_search": { + "filter": [ + "word_delimiter" + ], + "tokenizer": "classic" + } + } + }, + "number_of_shards": "1" + } + }, + "mappings": { + "dynamic_templates": [ + { + "strings": { + "mapping": { + "ignore_above": 256, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ] + }, + "aliases": {} +} + +PUT _ilm/policy/ilm_$[[INDEX_PREFIX]]metrics-30days-retention +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_age": "30d", + "max_size": "50gb" + }, + "set_priority": { + "priority": 100 + } + } + }, + "delete": { + "min_age": "30d", + "actions": { + "delete": { + } + } + } + } + } +} + +PUT _template/$[[INDEX_PREFIX]]metrics-rollover +{ + "order" : 100000, + "index_patterns" : [ + "$[[INDEX_PREFIX]]metrics*" + ], + "settings" : { + "index" : { + "format" : "7", + "lifecycle" : { + "name" : "ilm_$[[INDEX_PREFIX]]metrics-30days-retention", + "rollover_alias" : "$[[INDEX_PREFIX]]metrics" + }, + "codec" : "best_compression", + "number_of_shards" : "1", + "translog.durability":"async" + } + }, + "mappings" : { + "dynamic_templates" : [ + { + "strings" : { + "mapping" : { + "ignore_above" : 256, + "type" : "keyword" + }, + "match_mapping_type" : "string" + } + } + ] + }, + "aliases" : { } + } + + +PUT $[[INDEX_PREFIX]]metrics-00001 +{ + "settings": { + "index.lifecycle.rollover_alias":"$[[INDEX_PREFIX]]metrics" + , "refresh_interval": "5s" + }, + "aliases":{ + "$[[INDEX_PREFIX]]metrics":{ + "is_write_index":true + } + }, + "mappings": { + "dynamic_templates": [ + { + "strings": { + "match_mapping_type": "string", + "mapping": { + "ignore_above": 256, + "type": "keyword" + } + } + } + ], + "properties": { + "metadata": { + "properties": { + "category": { + "type": "keyword", + "ignore_above": 256 + }, + "datatype": { + "type": "keyword", + "ignore_above": 256 + }, + "labels": { + "properties": { + "cluster_id": { + "type": "keyword", + "ignore_above": 256 + }, + "index_id": { + "type": "keyword", + "ignore_above": 256 + }, + "index_name": { + "type": "keyword", + "ignore_above": 256 + }, + "index_uuid": { + "type": "keyword", + "ignore_above": 256 + }, + "ip": { + "type": "keyword", + "ignore_above": 256 + }, + "node_id": { + "type": "keyword", + "ignore_above": 256 + }, + "node_name": { + "type": "keyword", + "ignore_above": 256 + }, + "transport_address": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "name": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "timestamp": { + "type": "date" + } + } + } +} + + + +PUT _template/$[[INDEX_PREFIX]]alert-history-rollover +{ + "order" : 100000, + "index_patterns" : [ + "$[[INDEX_PREFIX]]alert-history*" + ], + "settings" : { + "index" : { + "format" : "7", + "lifecycle" : { + "name" : "ilm_$[[INDEX_PREFIX]]metrics-30days-retention", + "rollover_alias" : "$[[INDEX_PREFIX]]alert-history" + }, + "codec" : "best_compression", + "number_of_shards" : "1", + "translog.durability":"async" + } + }, + "mappings" : { + "dynamic_templates" : [ + { + "strings" : { + "mapping" : { + "ignore_above" : 256, + "type" : "keyword" + }, + "match_mapping_type" : "string" + } + } + ] + }, + "aliases" : { } + } + + +PUT $[[INDEX_PREFIX]]alert-history-00001 +{ + "settings": { + "index.lifecycle.rollover_alias":"$[[INDEX_PREFIX]]alert-history" + , "refresh_interval": "5s" + }, + "aliases":{ + "$[[INDEX_PREFIX]]alert-history":{ + "is_write_index":true + } + }, + "mappings": { + "properties" : { + "condition" : { + "properties" : { + "items" : { + "properties" : { + "expression" : { + "type" : "keyword", + "ignore_above" : 256 + }, + "minimum_period_match" : { + "type" : "long" + }, + "operator" : { + "type" : "keyword", + "ignore_above" : 256 + }, + "severity" : { + "type" : "keyword", + "ignore_above" : 256 + }, + "values" : { + "type" : "keyword", + "ignore_above" : 256 + } + } + }, + "operator" : { + "type" : "keyword", + "ignore_above" : 256 + } + } + }, + "condition_result" : { + "type" : "object", + "enabled" : false + }, + "context" : { + "type" : "keyword", + "copy_to" : [ + "search_text" + ] + }, + "created" : { + "type" : "date" + }, + "expression" : { + "type" : "keyword", + "copy_to" : [ + "search_text" + ] + }, + "id" : { + "type" : "keyword" + }, + "is_escalated" : { + "type" : "boolean" + }, + "is_notified" : { + "type" : "boolean" + }, + "message" : { + "type" : "keyword", + "ignore_above" : 256 + }, + "objects" : { + "type" : "keyword", + "copy_to" : [ + "search_text" + ] + }, + "resource_id" : { + "type" : "keyword" + }, + "resource_name" : { + "type" : "keyword" + }, + "rule_id" : { + "type" : "keyword" + }, + "rule_name" : { + "type" : "keyword" + }, + "search_text" : { + "type" : "text", + "analyzer" : "suggest_text_search", + "index_prefixes" : { + "min_chars" : 2, + "max_chars" : 5 + }, + "index_phrases" : true + }, + "severity" : { + "type" : "keyword" + }, + "state" : { + "type" : "keyword", + "ignore_above" : 256 + }, + "title" : { + "type" : "keyword" + }, + "updated" : { + "type" : "date" + } + } + } +} + + +PUT _template/$[[INDEX_PREFIX]]activities-rollover +{ + "order" : 100000, + "index_patterns" : [ + "$[[INDEX_PREFIX]]activities*" + ], + "settings" : { + "index" : { + "format" : "7", + "lifecycle" : { + "name" : "ilm_$[[INDEX_PREFIX]]metrics-30days-retention", + "rollover_alias" : "$[[INDEX_PREFIX]]activities" + }, + "codec" : "best_compression", + "number_of_shards" : "1", + "translog.durability":"async" + } + }, + "mappings" : { + "dynamic_templates" : [ + { + "strings" : { + "mapping" : { + "ignore_above" : 256, + "type" : "keyword" + }, + "match_mapping_type" : "string" + } + } + ] + }, + "aliases" : { } + } + + +PUT $[[INDEX_PREFIX]]activities-00001 +{ + "mappings": { + "dynamic_templates": [ + { + "strings": { + "match_mapping_type": "string", + "mapping": { + "ignore_above": 256, + "type": "keyword" + } + } + } + ], + "properties": { + "changelog": { + "type": "flattened" + }, + "id": { + "type": "keyword" + }, + "metadata": { + "properties": { + "category": { + "type": "keyword", + "ignore_above": 256 + }, + "group": { + "type": "keyword", + "ignore_above": 256 + }, + "labels": { + "type": "flattened" + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "type": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "payload": { + "type": "object", + "enabled": false + }, + "timestamp": { + "type": "date" + } + } + }, + "settings": { + "index": { + "lifecycle.rollover_alias": "$[[INDEX_PREFIX]]activities", + "refresh_interval": "5s", + "mapping": { + "total_fields": { + "limit": "20000" + } + }, + "max_result_window": "10000000", + "analysis": { + "analyzer": { + "suggest_text_search": { + "filter": [ + "word_delimiter" + ], + "tokenizer": "classic" + } + } + } + } + }, + "aliases": { + "$[[INDEX_PREFIX]]activities": { + "is_write_index": true + } + } +} + + +#alerting +POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calakp97h710dpnp1fa2 +{ + "id": "builtin-calakp97h710dpnp1fa2", + "created": "2022-06-16T03:58:29.437447113Z", + "updated": "2022-07-21T23:12:51.111569117Z", + "name": "CPU utilization is Too High", + "enabled": false, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + "$[[INDEX_PREFIX]]metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "node_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.node_id", + "limit": 300 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.node_stats.process.cpu.percent", + "statistic": "avg" + } + ], + "format_type": "ratio", + "expression": "avg(payload.elasticsearch.node_stats.process.cpu.percent)", + "title": "CPU Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%", + "message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}};NodeID:{{index .group_values 1}}; CPU:{{.result_value | to_fixed 2}}%;\n{{end}}" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "80" + ], + "priority": "low" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "90" + ], + "priority": "medium" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "95" + ], + "priority": "high" + } + ] + }, + "channels": { + "enabled": true, + "normal": [ + { + "created": "2022-06-16T04:11:10.242061032Z", + "updated": "2022-06-16T04:11:10.242061032Z", + "name": "Slack", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "${SLACK_WEBHOOK_ENDPOINT}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" + } + } + ], + "throttle_period": "6h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "schedule": { + "interval": "1m" + } +} + +#The `id` value is consistent with the `_id` value +POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnoaps0 +{ + "id": "builtin-cal8n7p7h710dpnoaps0", + "created": "2022-06-16T01:47:11.326727124Z", + "updated": "2022-07-13T04:00:06.181994982Z", + "name": "Cluster Health Change to Red", + "enabled": false, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + "$[[INDEX_PREFIX]]metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "match": { + "payload.elasticsearch.cluster_health.status": "red" + } + }, + { + "term": { + "metadata.name": { + "value": "cluster_health" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.cluster_health.status", + "statistic": "count" + } + ], + "format_type": "num", + "expression": "count(payload.elasticsearch.cluster_health.status)", + "title": "Health of Cluster[s] ({{.first_group_value}} ..., {{len .results}} clusters in total) Changed to Red", + "message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}} is red now;\n{{end}}" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "critical" + } + ] + }, + "channels": { + "enabled": true, + "normal": [ + { + "created": "2022-06-16T01:47:11.326727124Z", + "updated": "2022-06-16T01:47:11.326727124Z", + "name": "Slack webhook", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "${SLACK_WEBHOOK_ENDPOINT}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"high\"}} \"#EB4C21\" {{else if eq .priority \"medium\"}} \"#FFB449\" {{else if eq .priority \"low\"}} \"#87d068\" {{else}} \"#2db7f5\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|View Cluster Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" + } + }, + { + "created": "2022-06-16T01:47:11.326727124Z", + "updated": "2022-06-16T01:47:11.326727124Z", + "name": "DingTalk", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "url": "${DINGTALK_WEBHOOK_ENDPOINT}", + "body": "{\"msgtype\": \"text\",\"text\": {\"content\":\"Alerting: \\n{{.title}}\\n\\n{{.message}}\\nLink:${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}\"}}" + } + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "schedule": { + "interval": "1m" + } +} + + +#The `id` value is consistent with the `_id` value +POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cal8n7p7h710dpnogps1 +{ + "id": "builtin-cal8n7p7h710dpnogps1", + "created": "2022-06-16T03:11:01.445958361Z", + "updated": "2022-07-22T00:06:26.498903821Z", + "name": "Disk utilization is Too High", + "enabled": false, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + "$[[INDEX_PREFIX]]metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "node_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.node_id", + "limit": 200 + } + ], + "formula": "((a-b)/a)*100", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.node_stats.fs.data.total_in_bytes", + "statistic": "max" + }, + { + "name": "b", + "field": "payload.elasticsearch.node_stats.fs.data.free_in_bytes", + "statistic": "max" + } + ], + "format_type": "ratio", + "expression": "((max(payload.elasticsearch.node_stats.fs.data.total_in_bytes)-max(payload.elasticsearch.node_stats.fs.data.free_in_bytes))/max(payload.elasticsearch.node_stats.fs.data.total_in_bytes))*100", + "title": "Disk Utilization is Too High", + "message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID：{{index .group_values 0}} ;\nNodeID：{{index .group_values 1}} ;\nDisk Usage:{{.result_value | to_fixed 2}}%；Free Storage:{{.relation_values.b | format_bytes 2}}；\n{{end}}" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 5, + "operator": "gte", + "values": [ + "85" + ], + "priority": "low" + }, + { + "minimum_period_match": 5, + "operator": "gte", + "values": [ + "90" + ], + "priority": "medium" + }, + { + "minimum_period_match": 5, + "operator": "gte", + "values": [ + "95" + ], + "priority": "high" + } + ] + }, + "channels": { + "enabled": true, + "normal": [ + { + "created": "0001-01-01T00:00:00Z", + "updated": "0001-01-01T00:00:00Z", + "name": "Slack", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "${SLACK_WEBHOOK_ENDPOINT}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Free:* {{.relation_values.b | format_bytes 2}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" + } + } + ], + "throttle_period": "3h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "schedule": { + "interval": "1m" + } +} + +#The `id` value is consistent with the `_id` value +POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cbp20n2anisjmu4gehc5 +{ + "id": "builtin-cbp20n2anisjmu4gehc5", + "created": "2022-08-09T08:52:44.63345561Z", + "updated": "2022-08-09T08:52:44.633455664Z", + "name": "Elasticsearch node left cluster", + "enabled": false, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + "$[[INDEX_PREFIX]]node" + ], + "filter": {}, + "raw_filter": { + "match_phrase": { + "metadata.labels.status": "unavailable" + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.cluster_id", + "limit": 5 + }, + { + "field": "metadata.node_id", + "limit": 50 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "metadata.labels.status", + "statistic": "count" + } + ], + "format_type": "num", + "expression": "count(metadata.labels.status)", + "title": "Elasticsearch node left cluster", + "message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nNodeID:{{index .group_values 1}}; \n{{end}}" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "critical" + } + ] + }, + "channels": { + "enabled": true, + "normal": [ + { + "created": "2022-08-09T08:52:44.63345561Z", + "updated": "2022-08-09T08:52:44.63345561Z", + "name": "Wechat", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "${WECHAT_WEBHOOK_ENDPOINT}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"Incident [#{{.event_id}}](${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}) is ongoing\\n{{.title}}\\n\n {{range .results}}\n >ClusterID:{{index .group_values 0}}\n >NodeID:{{index .group_values 1}}\n >Priority:{{.priority}}\n >Link:[View Cluster Monitoring](${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}) \n {{end}}\"\n }\n}\n" + } + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "schedule": { + "interval": "1m" + } +} + + +#The `id` value is consistent with the `_id` value +POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calavvp7h710dpnp32r3 +{ + "id": "builtin-calavvp7h710dpnp32r3", + "created": "2022-06-16T04:22:23.001354546Z", + "updated": "2022-07-21T23:10:36.70696738Z", + "name": "Index Health Change to Red", + "enabled": false, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + "$[[INDEX_PREFIX]]index" + ], + "filter": {}, + "raw_filter": { + "match_phrase": { + "metadata.labels.health_status": "red" + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.cluster_id", + "limit": 5 + }, + { + "field": "metadata.index_name", + "limit": 5 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "metadata.index_name", + "statistic": "count" + } + ], + "format_type": "num", + "expression": "count(metadata.index_name)", + "title": "Health of Indices ({{.first_group_value}} ..., {{len .results}} indices in total) Changed to Red", + "message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; Index name:{{index .group_values 1}}; {{end}}" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "high" + } + ] + }, + "channels": { + "enabled": true, + "normal": [ + { + "created": "2022-06-16T04:11:10.242061032Z", + "updated": "2022-06-16T04:11:10.242061032Z", + "name": "Slack", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "${SLACK_WEBHOOK_ENDPOINT}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}?_g=%7B%22tab%22%3A%22indices%22%7D|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" + } + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "schedule": { + "interval": "1m" + } +} + + +#The `id` value is consistent with the `_id` value +POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calaqnh7h710dpnp2bm8 +{ + "id": "builtin-calaqnh7h710dpnp2bm8", + "created": "2022-06-16T04:11:10.242061032Z", + "updated": "2022-07-21T23:12:07.142532243Z", + "name": "JVM utilization is Too High", + "enabled": false, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + "$[[INDEX_PREFIX]]metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "node_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.node_id", + "limit": 300 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.node_stats.jvm.mem.heap_used_percent", + "statistic": "p90" + } + ], + "format_type": "ratio", + "expression": "p90(payload.elasticsearch.node_stats.jvm.mem.heap_used_percent)", + "title": "JVM Usage of Node[s] ({{.first_group_value}} ..., {{len .results}} nodes in total) >= {{.first_threshold}}%", + "message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; Node name:{{index .group_values 1}}; memory used percent：{{.result_value | to_fixed 2}}%;{{end}}" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "80" + ], + "priority": "low" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "90" + ], + "priority": "medium" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "95" + ], + "priority": "high" + } + ] + }, + "channels": { + "enabled": true, + "normal": [ + { + "created": "2022-06-16T04:11:10.242061032Z", + "updated": "2022-06-16T04:11:10.242061032Z", + "name": "Slack", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "${SLACK_WEBHOOK_ENDPOINT}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*NodeID:* {{index .group_values 1}}\"\n }\n ,\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Usage:* {{.result_value | to_fixed 2}}%\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/nodes/{{ index .group_values 1}}|View Node Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" + } + } + ], + "throttle_period": "3h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "schedule": { + "interval": "1m" + } +} + +#The `id` value is consistent with the `_id` value +POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-cbp2e4ianisjmu4giqs7 +{ + "id": "builtin-cbp2e4ianisjmu4giqs7", + "created": "2022-06-16T04:11:10.242061032Z", + "updated": "2022-08-09T09:39:29.604751601Z", + "name": "Search latency is great than 500ms", + "enabled": false, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + "$[[INDEX_PREFIX]]metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "index_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ], + "must_not": [ + { + "term": { + "metadata.labels.index_name": { + "value": "_all" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.index_name", + "limit": 500 + } + ], + "formula": "a/b", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.index_stats.total.search.query_time_in_millis", + "statistic": "rate" + }, + { + "name": "b", + "field": "payload.elasticsearch.index_stats.primaries.search.query_total", + "statistic": "rate" + } + ], + "format_type": "num", + "expression": "rate(payload.elasticsearch.index_stats.total.search.query_time_in_millis)/rate(payload.elasticsearch.index_stats.primaries.search.query_total)", + "title": "Search latency is great than 500ms", + "message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nIndex name:{{index .group_values 1}}; \nCurrent value:{{.result_value | to_fixed 2}}ms;\n{{end}}" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "500" + ], + "priority": "medium" + } + ] + }, + "channels": { + "enabled": true, + "normal": [ + { + "created": "2022-06-16T04:11:10.242061032Z", + "updated": "2022-06-16T04:11:10.242061032Z", + "name": "Slack", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "${SLACK_WEBHOOK_ENDPOINT}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Latency:* {{.result_value | to_fixed 2}}ms\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" + } + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "schedule": { + "interval": "1m" + } +} + +#The `id` value is consistent with the `_id` value +POST $[[INDEX_PREFIX]]alert-rule/_doc/builtin-calgapp7h710dpnpbeb6 +{ + "id": "builtin-calgapp7h710dpnpbeb6", + "created": "2022-06-16T10:26:47.360988761Z", + "updated": "2022-07-22T00:03:34.044562893Z", + "name": "Shard Storage >= 55G", + "enabled": false, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + "$[[INDEX_PREFIX]]metrics*" + ], + "filter": {}, + "raw_filter": { + "range": { + "payload.elasticsearch.index_stats.shard_info.store_in_bytes": { + "gte": 59055800320 + } + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.index_name", + "limit": 500 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.index_stats.shard_info.store_in_bytes", + "statistic": "max" + } + ], + "format_type": "bytes", + "expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)", + "title": "Shard Storage >55GB in ({{.first_group_value}} ..., {{len .results}} indices in total)", + "message": "Timestamp:{{.timestamp | datetime}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; Index:{{index .group_values 1}}; Max Shard Storage：{{.result_value | format_bytes 2}};{{end}}" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "53687091200" + ], + "priority": "high" + } + ] + }, + "channels": { + "enabled": true, + "normal": [ + { + "created": "2022-06-16T04:11:10.242061032Z", + "updated": "2022-06-16T04:11:10.242061032Z", + "name": "Slack", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "${SLACK_WEBHOOK_ENDPOINT}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Max Shard Storage:* {{.result_value | format_bytes 2}}\"\n },\n \n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" + } + } + ], + "throttle_period": "24h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "schedule": { + "interval": "1m" + } +} + +#The `id` value is consistent with the `_id` value +POST $[[INDEX_PREFIX]]alert-rule/_doc/cb34sfl6psfiqtovhpt4 +{ + "id": "cb34sfl6psfiqtovhpt4", + "created": "2022-07-07T03:08:46.297166036Z", + "updated": "2022-08-09T08:40:05.323148338Z", + "name": "Too Many Deleted Documents", + "enabled": false, + "resource": { + "resource_id": "$[[RESOURCE_ID]]", + "resource_name": "$[[RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + "$[[INDEX_PREFIX]]metrics*" + ], + "filter": {}, + "raw_filter": { + "range": { + "payload.elasticsearch.cluster_stats.indices.store.size_in_bytes": { + "gte": 32212254720 + } + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.index_name", + "limit": 300 + } + ], + "formula": "(a/(a+b))*100", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.index_stats.primaries.docs.deleted", + "statistic": "max" + }, + { + "name": "b", + "field": "payload.elasticsearch.index_stats.primaries.docs.count", + "statistic": "max" + } + ], + "format_type": "ratio", + "expression": "(max(payload.elasticsearch.index_stats.primaries.docs.deleted)/(max(payload.elasticsearch.index_stats.primaries.docs.deleted)+max(payload.elasticsearch.index_stats.primaries.docs.count)))*100", + "title": "Too Many Deleted Documents (>30%)", + "message": "Priority:{{.priority}}\nTimestamp:{{.timestamp | datetime_in_zone \"Asia/Shanghai\"}}\nRuleID:{{.rule_id}}\nEventID:{{.event_id}}\n{{range .results}}\nClusterID:{{index .group_values 0}}; \nIndex:{{index .group_values 0}}; \nRatio of Deleted Documents:{{.result_value}};\n{{end}}" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "30" + ], + "priority": "medium" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "40" + ], + "priority": "high" + } + ] + }, + "channels": { + "enabled": true, + "normal": [ + { + "created": "2022-06-16T04:11:10.242061032Z", + "updated": "2022-06-16T04:11:10.242061032Z", + "name": "Slack", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "${SLACK_WEBHOOK_ENDPOINT}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Incident <${INFINI_CONSOLE_ENDPOINT}/#/alerting/alert/{{.event_id}}|#{{.event_id}}> is ongoing\\n{{.title}}\"\n }\n }\n ],\n \"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"fields\": [\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*ClusterID:* {{index .group_values 0}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Index:* {{index .group_values 1}}\"\n },\n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Deleted:* {{.result_value | to_fixed 2}}%\"\n },\n \n {\n \"type\": \"mrkdwn\",\n \"text\": \"*Link:* <${INFINI_CONSOLE_ENDPOINT}/#/cluster/overview/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22cluster_name%22:%22{{ index .group_values 0}}%22}|View Index Monitoring>\"\n }\n ]\n }\n ]\n },\n {{end}}\n ]\n}" + } + } + ], + "throttle_period": "24h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "schedule": { + "interval": "1m" + } +} + + +GET / + + diff --git a/config/system_config.tpl b/config/system_config.tpl new file mode 100644 index 00000000..e4ae273e --- /dev/null +++ b/config/system_config.tpl @@ -0,0 +1,84 @@ + +elasticsearch: + - id: $[[CLUSTER_ID]] + name: $[[CLUSTER_ID]] + enabled: true + monitored: true + reserved: true + endpoint: $[[CLUSTER_ENDPINT]] + basic_auth: + username: $[[CLUSTER_USER]] + password: $[[CLUSTER_PASS]] + +elastic.elasticsearch: $[[CLUSTER_ID]] + +pipeline: + - name: indexing_merge + auto_start: true + keep_running: true + processor: + - indexing_merge: + input_queue: "metrics" + elasticsearch: "$[[CLUSTER_ID]]" + index_name: "$[[INDEX_PREFIX]]metrics" + output_queue: + name: "metrics_requests" + label: + tag: "metrics" + worker_size: 1 + bulk_size_in_mb: 5 + - name: consume-metrics_requests + auto_start: true + keep_running: true + processor: + - bulk_indexing: + bulk: + compress: true + batch_size_in_mb: 5 + batch_size_in_docs: 5000 + consumer: + fetch_max_messages: 100 + queues: + type: indexing_merge + when: + cluster_available: ["$[[CLUSTER_ID]]"] + - name: metadata_ingest + auto_start: true + keep_running: true + processor: + - metadata: + bulk_size_in_mb: 5 + bulk_max_docs_count: 5000 + fetch_max_messages: 100 + elasticsearch: "$[[CLUSTER_ID]]" + queues: + type: metadata + category: elasticsearch + consumer: + group: metadata + when: + cluster_available: ["$[[CLUSTER_ID]]"] + - name: activity_ingest + auto_start: true + keep_running: true + processor: + - activity: + bulk_size_in_mb: 5 + bulk_max_docs_count: 5000 + fetch_max_messages: 100 + elasticsearch: "$[[CLUSTER_ID]]" + queues: + category: elasticsearch + activity: true + consumer: + group: activity + when: + cluster_available: ["$[[CLUSTER_ID]]"] + - name: cluster_migration_split + auto_start: true + keep_running: true + processor: + - cluster_migration: + elasticsearch: "$[[CLUSTER_ID]]" + when: + cluster_available: ["$[[CLUSTER_ID]]"] \ No newline at end of file diff --git a/console.yml b/console.yml index 7f24d827..23116695 100644 --- a/console.yml +++ b/console.yml @@ -1,20 +1,10 @@ -# for the system cluster, please use Elasticsearch v7.3+ -elasticsearch: - - name: default - enabled: true - monitored: false - endpoint: http://localhost:9200 - basic_auth: - username: elastic - password: infinilabs - discovery: - enabled: true +path.configs: "config" web: enabled: true embedding_api: true auth: - enabled: false + enabled: true ui: enabled: true path: .public @@ -27,7 +17,6 @@ web: enabled: true elastic: - elasticsearch: default enabled: true remote_configs: true health_check: @@ -58,67 +47,4 @@ metrics: enabled: true cluster_stats: true node_stats: true - index_stats: true - -pipeline: - - name: indexing_merge - auto_start: true - keep_running: true - processor: - - indexing_merge: - input_queue: "metrics" - elasticsearch: "default" - index_name: ".infini_metrics" - output_queue: - name: "metrics_requests" - label: - tag: "metrics" - worker_size: 1 - bulk_size_in_mb: 10 - - name: consume-metrics_requests - auto_start: true - keep_running: true - processor: - - bulk_indexing: - bulk: - compress: true - batch_size_in_mb: 10 - batch_size_in_docs: 5000 - consumer: - fetch_max_messages: 100 - queues: - type: indexing_merge - when: - cluster_available: ["default"] - - name: metadata_ingest - auto_start: true - keep_running: true - processor: - - metadata: - bulk_size_in_mb: 10 - bulk_max_docs_count: 5000 - fetch_max_messages: 1000 - elasticsearch: "default" - queues: - type: metadata - category: elasticsearch - consumer: - group: metadata - when: - cluster_available: ["default"] - - name: activity_ingest - auto_start: true - keep_running: true - processor: - - activity: - bulk_size_in_mb: 10 - bulk_max_docs_count: 5000 - fetch_max_messages: 1000 - elasticsearch: "default" - queues: - category: elasticsearch - activity: true - consumer: - group: activity - when: - cluster_available: ["default"] + index_stats: true \ No newline at end of file diff --git a/main.go b/main.go index 8c31eadb..6611bae3 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "errors" _ "expvar" log "github.com/cihub/seelog" @@ -12,11 +13,12 @@ import ( "infini.sh/framework" "infini.sh/framework/core/elastic" "infini.sh/framework/core/env" + "infini.sh/framework/core/global" "infini.sh/framework/core/insight" _ "infini.sh/framework/core/log" "infini.sh/framework/core/module" "infini.sh/framework/core/orm" - task2 "infini.sh/framework/core/task" + task1 "infini.sh/framework/core/task" "infini.sh/framework/modules/agent" _ "infini.sh/framework/modules/api" elastic2 "infini.sh/framework/modules/elastic" @@ -25,10 +27,13 @@ import ( "infini.sh/framework/modules/pipeline" queue2 "infini.sh/framework/modules/queue/disk_queue" "infini.sh/framework/modules/redis" + "infini.sh/framework/modules/security" "infini.sh/framework/modules/stats" "infini.sh/framework/modules/task" "infini.sh/framework/modules/ui" _ "infini.sh/framework/plugins" + setup1 "infini.sh/console/plugin/setup" + _ "infini.sh/console/plugin" api2 "infini.sh/gateway/api" _ "infini.sh/gateway/proxy" _ "time/tzdata" @@ -38,12 +43,7 @@ var appConfig *config.AppConfig var appUI *UI func main() { - terminalHeader := ("\n\n") - terminalHeader += (" ___ __ ___ ___ \n") - terminalHeader += (" / __\\/ / /___\\/\\ /\\ / \\ \n") - terminalHeader += (" / / / / // // / \\ \\/ /\\ / \n") - terminalHeader += ("/ /__/ /__/ \\_//\\ \\_/ / /_// \n") - terminalHeader += ("\\____|____|___/ \\___/___,' \n") + terminalHeader := ("\n") terminalHeader += (" ___ ___ __ __ ___ __ __ \n") terminalHeader += (" / __\\/___\\/\\ \\ \\/ _\\ /___\\/ / /__\\\n") terminalHeader += (" / / // // \\/ /\\ \\ // // / /_\\ \n") @@ -53,7 +53,7 @@ func main() { terminalFooter := "" - app := framework.NewApp("console", "INFINI Cloud Console, The easiest way to operate your own elasticsearch platform.", + app := framework.NewApp("console", "The easiest way to operate your own search platform.", config.Version, config.BuildNumber, config.LastCommitLog, config.BuildDate, config.EOLDate, terminalHeader, terminalFooter) app.Init(nil) @@ -61,29 +61,51 @@ func main() { api := api2.GatewayAPI{} + modules:=[]module.Module{} + modules=append(modules,&stats.SimpleStatsModule{}) + modules=append(modules,&elastic2.ElasticModule{}) + modules=append(modules,&queue2.DiskQueue{}) + modules=append(modules,&redis.RedisModule{}) + modules=append(modules,&pipeline.PipeModule{}) + modules=append(modules,&task.TaskModule{}) + modules=append(modules,&agent.AgentModule{}) + modules=append(modules,&metrics.MetricsModule{}) + modules=append(modules,&security.Module{}) + modules=append(modules,&migration.MigrationModule{}) + + uiModule:=&ui.UIModule{} + if app.Setup(func() { - err := bootstrapRequirementCheck() - if err != nil { - panic(err) - } //load core modules first - module.RegisterSystemModule(&elastic2.ElasticModule{}) - module.RegisterSystemModule(&stats.SimpleStatsModule{}) - module.RegisterSystemModule(&queue2.DiskQueue{}) - module.RegisterSystemModule(&redis.RedisModule{}) - module.RegisterSystemModule(&ui.UIModule{}) - module.RegisterSystemModule(&pipeline.PipeModule{}) - module.RegisterSystemModule(&task.TaskModule{}) - module.RegisterSystemModule(&agent.AgentModule{}) - module.RegisterSystemModule(&migration.MigrationModule{}) + module.RegisterSystemModule(&setup1.Module{}) + module.RegisterSystemModule(uiModule) - module.RegisterUserPlugin(&metrics.MetricsModule{}) + var initFunc= func() { + module.RegisterSystemModule(&stats.SimpleStatsModule{}) + module.RegisterSystemModule(&elastic2.ElasticModule{}) + module.RegisterSystemModule(&queue2.DiskQueue{}) + module.RegisterSystemModule(&redis.RedisModule{}) + module.RegisterSystemModule(&pipeline.PipeModule{}) + module.RegisterSystemModule(&task.TaskModule{}) + module.RegisterSystemModule(&agent.AgentModule{}) + module.RegisterSystemModule(&metrics.MetricsModule{}) + module.RegisterSystemModule(&security.Module{}) + module.RegisterSystemModule(&migration.MigrationModule{}) + } + + if !global.Env().SetupRequired(){ + initFunc() + }else{ + for _, v := range modules { + v.Setup() + } + setup1.RegisterSetupCallback(initFunc) + } api.RegisterAPI("") appConfig = &config.AppConfig{ - Elasticsearch: "default", UI: config.UIConfig{ LocalPath: ".public", VFSEnabled: true, @@ -107,28 +129,51 @@ func main() { module.Start() - //orm.RegisterSchemaWithIndexName(model.Dict{}, "dict") - //orm.RegisterSchemaWithIndexName(model.Reindex{}, "reindex") - orm.RegisterSchemaWithIndexName(elastic.View{}, "view") - orm.RegisterSchemaWithIndexName(elastic.CommonCommand{}, "commands") - //orm.RegisterSchemaWithIndexName(elastic.TraceTemplate{}, "trace-template") - orm.RegisterSchemaWithIndexName(gateway.Instance{}, "gateway-instance") - orm.RegisterSchemaWithIndexName(alerting.Rule{}, "alert-rule") - orm.RegisterSchemaWithIndexName(alerting.Alert{}, "alert-history") - orm.RegisterSchemaWithIndexName(alerting.AlertMessage{}, "alert-message") - orm.RegisterSchemaWithIndexName(alerting.Channel{}, "channel") - orm.RegisterSchemaWithIndexName(insight.Visualization{}, "visualization") - orm.RegisterSchemaWithIndexName(insight.Dashboard{}, "dashboard") - orm.RegisterSchemaWithIndexName(task2.Task{}, "task") - orm.RegisterSchemaWithIndexName(task2.Log{}, "task-log") - api.RegisterSchema() - - go func() { - err := alerting2.InitTasks() - if err != nil { - log.Errorf("init alerting task error: %v", err) + var initFunc= func() { + if global.Env().SetupRequired() { + for _, v := range modules { + v.Start() + } } - }() + + elastic2.InitTemplate(false) + + //orm.RegisterSchemaWithIndexName(model.Dict{}, "dict") + orm.RegisterSchemaWithIndexName(elastic.View{}, "view") + orm.RegisterSchemaWithIndexName(elastic.CommonCommand{}, "commands") + //orm.RegisterSchemaWithIndexName(elastic.TraceTemplate{}, "trace-template") + orm.RegisterSchemaWithIndexName(gateway.Instance{}, "gateway-instance") + orm.RegisterSchemaWithIndexName(alerting.Rule{}, "alert-rule") + orm.RegisterSchemaWithIndexName(alerting.Alert{}, "alert-history") + orm.RegisterSchemaWithIndexName(alerting.AlertMessage{}, "alert-message") + orm.RegisterSchemaWithIndexName(alerting.Channel{}, "channel") + orm.RegisterSchemaWithIndexName(insight.Visualization{}, "visualization") + orm.RegisterSchemaWithIndexName(insight.Dashboard{}, "dashboard") + orm.RegisterSchemaWithIndexName(task1.Task{}, "task") + orm.RegisterSchemaWithIndexName(task1.Log{}, "task-log") + api.RegisterSchema() + + task1.RunWithinGroup("initialize_alerting",func(ctx context.Context) error { + err := alerting2.InitTasks() + if err != nil { + log.Errorf("init alerting task error: %v", err) + } + return err + }) + } + + if !global.Env().SetupRequired(){ + initFunc() + }else{ + setup1.RegisterSetupCallback(initFunc) + } + + if !global.Env().SetupRequired(){ + err := bootstrapRequirementCheck() + if err != nil { + panic(err) + } + } }, nil) { app.Run() diff --git a/plugin/api/alerting/alert.go b/plugin/api/alerting/alert.go index e62579f9..3336a2e4 100644 --- a/plugin/api/alerting/alert.go +++ b/plugin/api/alerting/alert.go @@ -10,6 +10,7 @@ import ( "infini.sh/console/model/alerting" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" + "infini.sh/framework/core/global" "infini.sh/framework/core/orm" "infini.sh/framework/core/util" "net/http" @@ -109,7 +110,7 @@ func (h *AlertAPI) searchAlert(w http.ResponseWriter, req *http.Request, ps http } func (h *AlertAPI) getAlertStats(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - esClient := elastic.GetClient(h.Config.Elasticsearch) + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) queryDsl := util.MapStr{ "size": 0, "query": util.MapStr{ diff --git a/plugin/api/alerting/api.go b/plugin/api/alerting/api.go index 0e0627b2..6ce66dea 100644 --- a/plugin/api/alerting/api.go +++ b/plugin/api/alerting/api.go @@ -5,7 +5,6 @@ package alerting import ( - "infini.sh/console/config" "infini.sh/framework/core/api" "infini.sh/framework/core/api/rbac/enum" ) @@ -13,7 +12,6 @@ import ( type AlertAPI struct { api.Handler - Config *config.AppConfig } func (alert *AlertAPI) Init() { diff --git a/plugin/api/alerting/message.go b/plugin/api/alerting/message.go index ae98e2cb..d52d349d 100644 --- a/plugin/api/alerting/message.go +++ b/plugin/api/alerting/message.go @@ -11,6 +11,7 @@ import ( alerting2 "infini.sh/console/service/alerting" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" + "infini.sh/framework/core/global" "infini.sh/framework/core/kv" "infini.sh/framework/core/orm" "infini.sh/framework/core/util" @@ -82,7 +83,7 @@ func (h *AlertAPI) ignoreAlertMessage(w http.ResponseWriter, req *http.Request, } func (h *AlertAPI) getAlertMessageStats(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - esClient := elastic.GetClient(h.Config.Elasticsearch) + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) must := []util.MapStr{ { "terms": util.MapStr{ diff --git a/plugin/api/alerting/rule.go b/plugin/api/alerting/rule.go index ee89bc70..e28b07e4 100644 --- a/plugin/api/alerting/rule.go +++ b/plugin/api/alerting/rule.go @@ -14,6 +14,7 @@ import ( httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" "infini.sh/framework/core/event" + "infini.sh/framework/core/global" "infini.sh/framework/core/kv" "infini.sh/framework/core/orm" "infini.sh/framework/core/queue" @@ -462,7 +463,8 @@ func (alertAPI *AlertAPI) searchRule(w http.ResponseWriter, req *http.Request, p } func (alertAPI *AlertAPI) getRuleAlertMessageNumbers(ruleIDs []string) ( map[string]interface{},error) { - esClient := elastic.GetClient(alertAPI.Config.Elasticsearch) + + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) queryDsl := util.MapStr{ "size": 0, "query": util.MapStr{ @@ -513,7 +515,7 @@ func (alertAPI *AlertAPI) fetchAlertInfos(w http.ResponseWriter, req *http.Reque alertAPI.WriteJSON(w, util.MapStr{}, http.StatusOK) return } - esClient := elastic.GetClient(alertAPI.Config.Elasticsearch) + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) queryDsl := util.MapStr{ "_source": []string{"state", "rule_id"}, "sort": []util.MapStr{ diff --git a/plugin/api/gateway/api.go b/plugin/api/gateway/api.go index 23c6929c..3034a3a8 100644 --- a/plugin/api/gateway/api.go +++ b/plugin/api/gateway/api.go @@ -13,7 +13,7 @@ type GatewayAPI struct { api.Handler } -func init() { +func InitAPI() { gateway:=GatewayAPI{} api.HandleAPIMethod(api.POST, "/gateway/instance/try_connect", gateway.RequireLogin(gateway.tryConnect)) api.HandleAPIMethod(api.GET, "/gateway/instance/:instance_id", gateway.RequirePermission(gateway.getInstance, enum.PermissionGatewayInstanceRead)) diff --git a/plugin/api/index_management/common_command.go b/plugin/api/index_management/common_command.go index c5dd19c9..08e5bcf2 100644 --- a/plugin/api/index_management/common_command.go +++ b/plugin/api/index_management/common_command.go @@ -5,6 +5,7 @@ import ( log "github.com/cihub/seelog" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" + "infini.sh/framework/core/global" "infini.sh/framework/core/orm" "infini.sh/framework/core/util" "net/http" @@ -28,7 +29,7 @@ func (h *APIHandler) HandleAddCommonCommandAction(w http.ResponseWriter, req *ht reqParams.Created = time.Now() reqParams.ID = util.GetUUID() - esClient := elastic.GetClient(h.Config.Elasticsearch) + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) queryDSL :=[]byte(fmt.Sprintf(`{"size":1, "query":{"bool":{"must":{"match":{"title.keyword":"%s"}}}}}`, reqParams.Title)) var indexName = orm.GetIndexName(reqParams) @@ -73,7 +74,7 @@ func (h *APIHandler) HandleSaveCommonCommandAction(w http.ResponseWriter, req *h return } reqParams.ID = ps.ByName("cid") - esClient := elastic.GetClient(h.Config.Elasticsearch) + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) queryDSL :=[]byte(fmt.Sprintf(`{"size":1, "query":{"bool":{"must":{"match":{"title.keyword":"%s"}}}}}`, reqParams.Title)) var indexName = orm.GetIndexName(reqParams) @@ -133,7 +134,7 @@ func (h *APIHandler) HandleQueryCommonCommandAction(w http.ResponseWriter, req * } queryDSL = fmt.Sprintf(queryDSL, filterBuilder.String(), size, from) - esClient := elastic.GetClient(h.Config.Elasticsearch) + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) searchRes, err := esClient.SearchWithRawQueryDSL(orm.GetIndexName(elastic.CommonCommand{}), []byte(queryDSL)) if err != nil { @@ -149,7 +150,7 @@ func (h *APIHandler) HandleQueryCommonCommandAction(w http.ResponseWriter, req * func (h *APIHandler) HandleDeleteCommonCommandAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { resBody := map[string]interface{}{} id := ps.ByName("cid") - esClient := elastic.GetClient(h.Config.Elasticsearch) + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) delRes, err := esClient.Delete(orm.GetIndexName(elastic.CommonCommand{}), "", id, "wait_for") if err != nil { log.Error(err) diff --git a/plugin/api/index_management/elasticsearch.go b/plugin/api/index_management/elasticsearch.go index 50a2719c..ba815d6c 100644 --- a/plugin/api/index_management/elasticsearch.go +++ b/plugin/api/index_management/elasticsearch.go @@ -6,6 +6,7 @@ import ( httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" "infini.sh/framework/core/event" + "infini.sh/framework/core/global" "infini.sh/framework/core/host" "infini.sh/framework/core/orm" "infini.sh/framework/core/util" @@ -25,7 +26,7 @@ func (handler APIHandler) ElasticsearchOverviewAction(w http.ResponseWriter, req // clusterIDs = append(clusterIDs, key) // return true //}) - esClient := elastic.GetClient(handler.Config.Elasticsearch) + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) queryDsl := util.MapStr{ "size": 100, } @@ -110,7 +111,7 @@ func (handler APIHandler) ElasticsearchOverviewAction(w http.ResponseWriter, req } func (handler APIHandler) getLatestClusterMonitorData(clusterIDs []interface{}) (*elastic.SearchResponse, error){ - client := elastic.GetClient(handler.Config.Elasticsearch) + client := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) queryDSLTpl := `{ "size": %d, "query": { @@ -155,7 +156,7 @@ func (handler APIHandler) getLatestClusterMonitorData(clusterIDs []interface{}) } func (handler APIHandler) getMetricCount(indexName, field string, clusterIDs []interface{}) (interface{}, error){ - client := elastic.GetClient(handler.Config.Elasticsearch) + client := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) queryDSL := util.MapStr{ "size": 0, "aggs": util.MapStr{ @@ -182,7 +183,7 @@ func (handler APIHandler) getMetricCount(indexName, field string, clusterIDs []i } func (handler APIHandler) getLastActiveHostCount() (int, error){ - client := elastic.GetClient(handler.Config.Elasticsearch) + client := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) queryDSL := `{ "size": 0, "query": { diff --git a/plugin/api/index_management/index.go b/plugin/api/index_management/index.go index 99ee9422..9f6d637c 100644 --- a/plugin/api/index_management/index.go +++ b/plugin/api/index_management/index.go @@ -1,6 +1,8 @@ package index_management import ( + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/global" "net/http" "strconv" "strings" @@ -33,7 +35,7 @@ func (handler APIHandler) GetDictListAction(w http.ResponseWriter, req *http.Req if len(tags) > 3 { tags = tags[0:3] } - rel, err := model2.GetDictList(from, size, name, tags, handler.Config.Elasticsearch) + rel, err := model2.GetDictList(from, size, name, tags, global.MustLookupString(elastic.GlobalSystemElasticsearchID)) if err != nil { resp["error"] = err resp["status"] = false diff --git a/plugin/api/index_management/rebuild.go b/plugin/api/index_management/rebuild.go index 25799bb6..2853cd35 100644 --- a/plugin/api/index_management/rebuild.go +++ b/plugin/api/index_management/rebuild.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" log "github.com/cihub/seelog" + "infini.sh/framework/core/global" "infini.sh/framework/core/orm" "net/http" "strings" @@ -33,7 +34,7 @@ func (handler APIHandler) HandleReindexAction(w http.ResponseWriter, req *http.R //fmt.Println(reindexItem) typ := handler.GetParameter(req, "_type") - ID, err := reindex(handler.Config.Elasticsearch, reindexItem, typ) + ID, err := reindex(global.MustLookupString(elastic.GlobalSystemElasticsearchID), reindexItem, typ) if err != nil { log.Error(err) resResult["error"] = err @@ -94,7 +95,7 @@ func (handler APIHandler) HandleDeleteRebuildAction(w http.ResponseWriter, req * id := ps.ByName("id") var ids = []string{id} resBody := newResponseBody() - err := deleteTasksByIds(handler.Config.Elasticsearch, ids) + err := deleteTasksByIds(global.MustLookupString(elastic.GlobalSystemElasticsearchID), ids) if err != nil { log.Error(err) resBody["error"] = err @@ -111,7 +112,7 @@ func (handler APIHandler) HandleGetRebuildListAction(w http.ResponseWriter, req size = handler.GetIntOrDefault(req, "size", 10) name = handler.GetParameter(req, "name") resBody = newResponseBody() - esName = handler.Config.Elasticsearch + esName = global.MustLookupString(elastic.GlobalSystemElasticsearchID) ) esResp, err := model.GetRebuildList(esName, from, size, name) if err != nil { diff --git a/plugin/api/init.go b/plugin/api/init.go index 4e43870e..aff72102 100644 --- a/plugin/api/init.go +++ b/plugin/api/init.go @@ -3,7 +3,9 @@ package api import ( "infini.sh/console/config" "infini.sh/console/plugin/api/alerting" + "infini.sh/console/plugin/api/gateway" "infini.sh/console/plugin/api/index_management" + "infini.sh/console/plugin/api/insight" "infini.sh/framework/core/api" "infini.sh/framework/core/api/rbac/enum" "path" @@ -58,9 +60,10 @@ func Init(cfg *config.AppConfig) { //}) alertAPI := alerting.AlertAPI{ - Config: cfg, } alertAPI.Init() + gateway.InitAPI() + insight.InitAPI() } diff --git a/plugin/api/insight/api.go b/plugin/api/insight/api.go index cba24a23..f050a45e 100644 --- a/plugin/api/insight/api.go +++ b/plugin/api/insight/api.go @@ -10,7 +10,7 @@ type InsightAPI struct { api.Handler } -func init() { +func InitAPI() { insight := InsightAPI{} api.HandleAPIMethod(api.POST, "/elasticsearch/:id/visualization/metadata", insight.HandleGetMetadata) api.HandleAPIMethod(api.POST, "/elasticsearch/:id/visualization/data", insight.HandleGetMetricData) diff --git a/plugin/setup/setup.go b/plugin/setup/setup.go new file mode 100644 index 00000000..b4d7984c --- /dev/null +++ b/plugin/setup/setup.go @@ -0,0 +1,459 @@ +package task + +import ( + "bytes" + "fmt" + "golang.org/x/crypto/bcrypt" + "infini.sh/framework/core/api" + "infini.sh/framework/core/api/rbac" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/env" + "infini.sh/framework/core/errors" + "infini.sh/framework/core/global" + "infini.sh/framework/core/module" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/util" + elastic2 "infini.sh/framework/modules/elastic" + elastic1 "infini.sh/framework/modules/elastic/common" + elastic3 "infini.sh/framework/modules/elastic/api" + "infini.sh/framework/modules/security" + "infini.sh/framework/plugins/replay" + "io" + "net/http" + uri2 "net/url" + "path" + "runtime" + "github.com/valyala/fasttemplate" + log "github.com/cihub/seelog" + "time" +) + +type Module struct { + api.Handler +} + +func (module *Module) Name() string { + return "setup" +} + +func init() { + module.RegisterSystemModule(&Module{}) +} + +func (module *Module) Setup() { + + if !global.Env().SetupRequired(){ + return + } + + api.HandleAPIMethod(api.POST, "/setup/_validate", module.validate) + api.HandleAPIMethod(api.POST, "/setup/_initialize", module.initialize) + elastic3.InitTestAPI() +} + +var setupFinishedCallback= []func() {} +func RegisterSetupCallback(f func()) { + setupFinishedCallback=append(setupFinishedCallback,f) +} + +func InvokeSetupCallback() { + for _,v:=range setupFinishedCallback{ + v() + } +} + +func (module *Module) Start() error { + return nil +} +func (module *Module) Stop() error { + return nil +} + +type SetupRequest struct { + Cluster struct { + Host string `json:"host"` + Schema string `json:"schema"` + Endpoint string `json:"endpoint"` + Username string `json:"username"` + Password string `json:"password"` + } `json:"cluster"` + + BootstrapUsername string `json:"bootstrap_username"` + BootstrapPassword string `json:"bootstrap_password"` +} + +var tempID="infini_default_system_cluster" + +const VersionTooOld ="elasticsearch_version_too_old" +const IndicesExists ="elasticsearch_indices_exists" +const TemplateExists ="elasticsearch_template_exists" + +var cfg1 elastic1.ORMConfig + +func (module *Module) validate(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + + if !global.Env().SetupRequired(){ + module.WriteError(w, "setup not permitted", 500) + return + } + + success:=false + var err error + var errType string + var fixTips string + var code int + code=200 + defer func() { + + global.Env().CheckSetup() + + result := util.MapStr{} + result["success"]=success + + if r := recover(); r != nil { + var v string + switch r.(type) { + case error: + v = r.(error).Error() + case runtime.Error: + v = r.(runtime.Error).Error() + case string: + v = r.(string) + } + if v!=""{ + success=false + result["error"]=util.MapStr{ + "reason":v, + } + if errType!=""{ + result["type"]=errType + } + if fixTips!=""{ + result["fix_tips"]=fixTips + } + code=500 + } + } + module.WriteJSON(w, result, code) + }() + + + err, client,_ := module.initTempClient(r) + if err!=nil{ + panic(err) + } + + //validate version + version := client.GetVersion() + if version != "" { + ver := &util.Version{} + ver, err = util.ParseSemantic(version) + if err != nil { + panic(err) + } + if ver.Major() >= 7 { + if ver.Major() == 7 && ver.Minor() < 3 { + errType = VersionTooOld + panic(errors.Errorf("elasticsearch version(%v) should greater than v7.3", version)) + } + } else { + errType = VersionTooOld + panic(errors.Errorf("elasticsearch version(%v) should greater than v7.3", version)) + } + } + cfg1 = elastic1.ORMConfig{} + exist, err := env.ParseConfig("elastic.orm", &cfg1) + if exist && err != nil { + panic(err) + } + + if cfg1.IndexPrefix==""{ + cfg1.IndexPrefix=".infini_" + } + if cfg1.TemplateName==""{ + cfg1.TemplateName=".infini" + } + + //validate indices + indices, err := client.GetIndices(util.TrimSpaces(cfg1.IndexPrefix) + "*") + if err != nil { + panic(err) + } + + if indices != nil && len(*indices) > 0 { + buff := bytes.Buffer{} + tipBuff := bytes.Buffer{} + for k, _ := range *indices { + buff.WriteString(k) + buff.WriteString("\n") + + tipBuff.WriteString("DELETE ") + tipBuff.WriteString(k) + tipBuff.WriteString("\n") + } + errType = IndicesExists + fixTips=tipBuff.String() + panic(errors.Errorf("there are following indices exists in target elasticsearch: \n%v", buff.String())) + } + + ok, err := client.TemplateExists(cfg1.TemplateName) + if err != nil { + panic(err) + } + if ok { + errType = TemplateExists + fixTips="DELETE /_template/"+util.TrimSpaces(cfg1.TemplateName) + panic(errors.Errorf("there are following template already exists in target elasticsearch: \n%v", cfg1.TemplateName)) + } + + success = true +} +var cfg elastic.ElasticsearchConfig +func (module *Module) initTempClient(r *http.Request) (error, elastic.API,SetupRequest) { + request := SetupRequest{} + err := module.DecodeJSON(r, &request) + if err != nil { + return err,nil,request + } + + if request.Cluster.Endpoint==""&&request.Cluster.Host==""{ + panic("invalid configuration") + } + + if request.Cluster.Endpoint==""{ + if request.Cluster.Host!=""&&request.Cluster.Schema!=""{ + request.Cluster.Endpoint=fmt.Sprintf("%v://%v",request.Cluster.Schema,request.Cluster.Host) + } + } + + cfg = elastic.ElasticsearchConfig{ + Enabled: true, + Reserved: true, + Endpoint: request.Cluster.Endpoint, + BasicAuth: &elastic.BasicAuth{ + Username: request.Cluster.Username, + Password: request.Cluster.Password, + }, + } + + if cfg.Endpoint!=""&&cfg.Host==""{ + uri,err:=uri2.Parse(cfg.Endpoint) + if err!=nil{ + panic(err) + } + cfg.Host=uri.Host + cfg.Schema=uri.Scheme + } + + cfg.ID = tempID + cfg.Name = "INFINI_SYSTEM ("+util.PickRandomName()+")" + elastic.InitMetadata(&cfg, true) + client, err := elastic1.InitClientWithConfig(cfg) + if err != nil { + return err,nil,request + } + + elastic.UpdateConfig(cfg) + elastic.UpdateClient(cfg, client) + cfg.Version=client.GetVersion() + global.Register(elastic.GlobalSystemElasticsearchID,tempID) + + return err, client,request +} + +func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + if !global.Env().SetupRequired(){ + module.WriteError(w, "setup not permitted", 500) + return + } + + success:=false + var err error + var errType string + var fixTips string + var code int + code=200 + defer func() { + + global.Env().CheckSetup() + + result := util.MapStr{} + result["success"]=success + + if r := recover(); r != nil { + var v string + switch r.(type) { + case error: + v = r.(error).Error() + case runtime.Error: + v = r.(runtime.Error).Error() + case string: + v = r.(string) + } + if v!=""{ + success=false + result["error"]=util.MapStr{ + "reason":v, + } + if errType!=""{ + result["type"]=errType + } + if fixTips!=""{ + result["fix_tips"]=fixTips + } + code=500 + } + } + module.WriteJSON(w, result, code) + }() + + err, client,request := module.initTempClient(r) + if err!=nil{ + panic(err) + } + + if cfg1.IndexPrefix==""{ + cfg1.IndexPrefix=".infini_" + } + if cfg1.TemplateName==""{ + cfg1.TemplateName=".infini" + } + + if !cfg1.Enabled{ + cfg1.Enabled=true + } + + if !cfg1.InitTemplate{ + cfg1.InitTemplate=true + } + + cfg.Reserved=true + cfg.Monitored=true + + //处理ORM + handler := elastic2.ElasticORM{Client: client, Config:cfg1 } + orm.Register("elastic_setup_"+util.GetUUID(), handler) + + //处理模版 + elastic2.InitTemplate(true) + + //处理生命周期 + //TEMPLATE_NAME + //INDEX_PREFIX + dslTplFile:=path.Join(global.Env().GetConfigDir(),"initialization.tpl") + dslFile:=path.Join(global.Env().GetConfigDir(),"initialization.dsl") + + var dsl []byte + dsl,err=util.FileGetContent(dslTplFile) + if err!=nil{ + panic(err) + } + + var dslWriteSuccess=false + if len(dsl)>0{ + var tpl *fasttemplate.Template + tpl,err=fasttemplate.NewTemplate(string(dsl), "$[[", "]]") + if err!=nil{ + panic(err) + } + if tpl!=nil{ + output:=tpl.ExecuteFuncString(func(w io.Writer, tag string) (int, error) { + switch tag { + case "TEMPLATE_NAME": + return w.Write([]byte(cfg1.TemplateName)) + case "INDEX_PREFIX": + return w.Write([]byte(cfg1.IndexPrefix)) + case "RESOURCE_ID": + return w.Write([]byte(cfg.ID)) + case "RESOURCE_NAME": + return w.Write([]byte(cfg.Name)) + } + panic(errors.Errorf("unknown tag: %v",tag)) + }) + _,err=util.FilePutContent(dslFile,output) + if err!=nil{ + panic(err) + } + dslWriteSuccess=true + } + } + + if dslWriteSuccess{ + lines := util.FileGetLines(dslFile) + _,err,_:=replay.ReplayLines(pipeline.AcquireContext(),lines,cfg.Schema,cfg.Host) + if err!=nil{ + log.Error(err) + } + } + + + //处理索引 + elastic2.InitSchema() + //init security + security.InitSecurity() + + //保存默认集群 + err=orm.Save(&cfg) + if err!=nil{ + panic(err) + } + + if request.BootstrapUsername!=""&&request.BootstrapPassword!=""{ + //Save bootstrap user + user:=rbac.User{} + user.ID="default_user_"+request.BootstrapUsername + user.Name=request.BootstrapUsername + user.NickName=request.BootstrapUsername + var hash []byte + hash, err = bcrypt.GenerateFromPassword([]byte(request.BootstrapPassword), bcrypt.DefaultCost) + if err!=nil{ + panic(err) + } + + user.Password=string(hash) + role:=[]rbac.UserRole{} + role=append(role,rbac.UserRole{ + ID: rbac.RoleAdminName, + Name: rbac.RoleAdminName, + }) + user.Roles=role + err=orm.Save(&user) + if err!=nil{ + panic(err) + } + } + + + //save to local file + file:=path.Join(global.Env().GetConfigDir(),"system_config.yml") + _,err=util.FilePutContent(file,fmt.Sprintf("configs.template:\n - name: \"system\"\n path: ./config/system_config.tpl\n variable:\n " + + "CLUSTER_ID: %v\n CLUSTER_ENDPINT: \"%v\"\n " + + "CLUSTER_USER: \"%v\"\n CLUSTER_PASS: \"%v\"\n INDEX_PREFIX: \"%v\"", + tempID,cfg.Endpoint,cfg.BasicAuth.Username,cfg.BasicAuth.Password,cfg1.IndexPrefix )) + if err!=nil{ + panic(err) + } + + //处理 ILM + + //callback + InvokeSetupCallback() + + //disable builtin auth + err=api.DisableBuiltinUserAdmin() + if err!=nil{ + panic(err) + } + + //place setup lock file + setupLock:=path.Join(global.Env().GetDataDir(),".setup_lock") + _,err=util.FilePutContent(setupLock,time.Now().String()) + if err!=nil{ + panic(err) + } + + success=true + +} diff --git a/ui.go b/ui.go index 95b7a9cd..92524c93 100644 --- a/ui.go +++ b/ui.go @@ -6,11 +6,11 @@ import ( "net/http" log "github.com/cihub/seelog" + "infini.sh/console/config" + uiapi "infini.sh/console/plugin/api" "infini.sh/framework/core/api" "infini.sh/framework/core/util" "infini.sh/framework/core/vfs" - "infini.sh/console/config" - uiapi "infini.sh/console/plugin/api" ) type UI struct { @@ -31,7 +31,7 @@ func (h UI) InitUI() { // //api.HandleUIFunc("/config", func(w http.ResponseWriter, req *http.Request){ // if(strings.TrimSpace(apiEndpoint) == ""){ - // hostParts := strings.Split(req.Host, ":") + // hostParts := strings.Split(req.RemoteIP, ":") // apiEndpoint = fmt.Sprintf("%s//%s:%s", apiConfig.GetSchema(), hostParts[0], apiConfig.NetworkConfig.GetBindingPort()) // } // buf, _ := json.Marshal(util.MapStr{