Merge pull request 'System Setup' (#16) from setup into master

Jenkinsfile

@@ -32,7 +32,7 @@ pipeline {
 
                                 sh label: 'copy-license', script: 'cd /home/jenkins/go/src/infini.sh/console && cp ../framework/LICENSE bin && cat ../framework/NOTICE NOTICE > bin/NOTICE'
 
-                                sh label: 'copy-configs', script: 'cd /home/jenkins/go/src/infini.sh/console && mkdir -p bin/config && cp config/*.json bin/config'
+                                sh label: 'copy-configs', script: 'cd /home/jenkins/go/src/infini.sh/console && mkdir -p bin/config && cp config/*.json bin/config && cp config/*.yml bin/config && cp config/*.tpl bin/config'
 
                                 sh label: 'package-linux-amd64', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-amd64.tar.gz console-linux-amd64 console.yml  LICENSE NOTICE config'
                                 sh label: 'package-linux-386', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-386.tar.gz console-linux-386 console.yml  LICENSE NOTICE config'

bootstrap_check.go

@@ -8,14 +8,14 @@ import (
 	"fmt"
 	"github.com/buger/jsonparser"
 	log "github.com/cihub/seelog"
-	"infini.sh/console/config"
 	"infini.sh/framework/core/elastic"
 	"infini.sh/framework/core/env"
+	"infini.sh/framework/core/global"
 	"infini.sh/framework/core/util"
 )
 
 func bootstrapRequirementCheck() error{
-	err := checkElasticsearchRequire()
+	err := checkElasticsearchRequirements()
 	if err != nil {
 		return err
 	}
@@ -23,7 +23,7 @@ func bootstrapRequirementCheck() error{
 }
 
 
-func checkElasticsearchRequire() error{
+func checkElasticsearchRequirements() error{
 	log.Trace("start to check elasticsearch requirement")
 	var esConfigs = []elastic.ElasticsearchConfig{}
 	ok, err := env.ParseConfig("elasticsearch", &esConfigs)
@@ -33,27 +33,24 @@ func checkElasticsearchRequire() error{
 	if !ok {
 		return fmt.Errorf("elasticsearch config section not found")
 	}
-	appConfig = &config.AppConfig{
+
-		Elasticsearch: "default",
+	elasticsearchID:=global.Lookup(elastic.GlobalSystemElasticsearchID)
-	}
+
-	ok, err = env.ParseConfig("web", appConfig)
+	if  elasticsearchID == nil||elasticsearchID=="" {
-	if err != nil {
+		return fmt.Errorf("elasticsearch config in web section can not be empty")
-		return fmt.Errorf("parse web config section error: %v", err)
-	}
-	if !ok {
-		return fmt.Errorf("web config section not found")
-	}
-	if  appConfig.Elasticsearch == "" {
-		return fmt.Errorf("elasticsearch config of web section can not be empty")
 	}
+
+	esID:=elasticsearchID.(string)
+
 	var targetEsConfig *elastic.ElasticsearchConfig
 	for _, esConfig := range esConfigs {
-		if esConfig.Name == appConfig.Elasticsearch {
+		if esConfig.ID == esID||(esConfig.ID==""&&esConfig.Name==esID) {
 			targetEsConfig = &esConfig
 		}
 	}
+
 	if targetEsConfig == nil {
-		return fmt.Errorf("elasticsearch config named %s not found", appConfig.Elasticsearch)
+		return fmt.Errorf("elasticsearch config %s was not found", esID)
 	}
 	var req = util.NewGetRequest(targetEsConfig.Endpoint, nil)
 	if targetEsConfig.BasicAuth != nil {
@@ -65,6 +62,10 @@ func checkElasticsearchRequire() error{
 		return fmt.Errorf("check elasticsearch requirement error: %v", err)
 	}
 
+	if result==nil||result.Body==nil||len(result.Body)==0{
+		return fmt.Errorf("failed to retrive elasticsearch version info")
+	}
+
 	versionNumber, err := jsonparser.GetString(result.Body, "version", "number")
 	if err != nil {
 		return fmt.Errorf("check elasticsearch requirement error: %v, got response: %s", err, string(result.Body))

config/config.go

@@ -3,7 +3,6 @@ package config
 import "infini.sh/framework/core/config"
 
 type AppConfig struct {
-	Elasticsearch string `config:"elasticsearch"`
 	UI UIConfig `config:"ui"`
 	Network config.NetworkConfig `config:"network"`
 	TLSConfig   config.TLSConfig     `config:"tls"`

config/initialization.tpl

@@ -0,0 +1,325 @@
+PUT _template/$[[TEMPLATE_NAME]]
+{
+    "order": 0,
+    "index_patterns": [
+      "$[[INDEX_PREFIX]]*"
+    ],
+    "settings": {
+      "index": {
+        "max_result_window": "10000000",
+        "mapping": {
+          "total_fields": {
+            "limit": "20000"
+          }
+        },
+        "analysis": {
+          "analyzer": {
+            "suggest_text_search": {
+              "filter": [
+                "word_delimiter"
+              ],
+              "tokenizer": "classic"
+            }
+          }
+        },
+        "number_of_shards": "1"
+      }
+    },
+    "mappings": {
+      "dynamic_templates": [
+        {
+          "strings": {
+            "mapping": {
+              "ignore_above": 256,
+              "type": "keyword"
+            },
+            "match_mapping_type": "string"
+          }
+        }
+      ]
+    },
+    "aliases": {}
+}
+
+PUT _ilm/policy/ilm_$[[INDEX_PREFIX]]metrics-30days-retention
+{
+  "policy": {
+    "phases": {
+      "hot": {
+        "min_age": "0ms",
+        "actions": {
+          "rollover": {
+            "max_age": "30d",
+            "max_size": "50gb"
+          },
+          "set_priority": {
+            "priority": 100
+          }
+        }
+      },
+      "delete": {
+        "min_age": "30d",
+        "actions": {
+          "delete": {
+          }
+        }
+      }
+    }
+  }
+}
+
+PUT _template/$[[INDEX_PREFIX]]metrics-rollover
+{
+    "order" : 100000,
+    "index_patterns" : [
+      "$[[INDEX_PREFIX]]metrics*"
+    ],
+    "settings" : {
+      "index" : {
+        "format" : "7",
+        "lifecycle" : {
+          "name" : "ilm_$[[INDEX_PREFIX]]metrics-30days-retention",
+          "rollover_alias" : "$[[INDEX_PREFIX]]metrics"
+        },
+        "codec" : "best_compression",
+        "number_of_shards" : "1",
+        "translog.durability":"async"
+      }
+    },
+    "mappings" : {
+      "dynamic_templates" : [
+        {
+          "strings" : {
+            "mapping" : {
+              "ignore_above" : 256,
+              "type" : "keyword"
+            },
+            "match_mapping_type" : "string"
+          }
+        }
+      ]
+    },
+    "aliases" : { }
+  }
+
+
+PUT $[[INDEX_PREFIX]]metrics-00001
+{
+  "settings": {
+    "index.lifecycle.rollover_alias":"$[[INDEX_PREFIX]]metrics"
+    , "refresh_interval": "5s"
+  },
+  "aliases":{
+    "$[[INDEX_PREFIX]]metrics":{
+      "is_write_index":true
+    }
+  }
+}
+
+
+
+PUT _template/$[[INDEX_PREFIX]]alert-history-rollover
+{
+    "order" : 100000,
+    "index_patterns" : [
+      "$[[INDEX_PREFIX]]alert-history*"
+    ],
+    "settings" : {
+      "index" : {
+        "format" : "7",
+        "lifecycle" : {
+          "name" : "ilm_$[[INDEX_PREFIX]]metrics-30days-retention",
+          "rollover_alias" : "$[[INDEX_PREFIX]]alert-history"
+        },
+        "codec" : "best_compression",
+        "number_of_shards" : "1",
+        "translog.durability":"async"
+      }
+    },
+    "mappings" : {
+      "dynamic_templates" : [
+        {
+          "strings" : {
+            "mapping" : {
+              "ignore_above" : 256,
+              "type" : "keyword"
+            },
+            "match_mapping_type" : "string"
+          }
+        }
+      ]
+    },
+    "aliases" : { }
+  }
+
+
+PUT $[[INDEX_PREFIX]]alert-history-00001
+{
+  "settings": {
+    "index.lifecycle.rollover_alias":"$[[INDEX_PREFIX]]alert-history"
+    , "refresh_interval": "5s"
+  },
+  "aliases":{
+    "$[[INDEX_PREFIX]]alert-history":{
+      "is_write_index":true
+    }
+  },
+  "mappings": {
+    "properties" : {
+        "condition" : {
+          "properties" : {
+            "items" : {
+              "properties" : {
+                "expression" : {
+                  "type" : "keyword",
+                  "ignore_above" : 256
+                },
+                "minimum_period_match" : {
+                  "type" : "long"
+                },
+                "operator" : {
+                  "type" : "keyword",
+                  "ignore_above" : 256
+                },
+                "severity" : {
+                  "type" : "keyword",
+                  "ignore_above" : 256
+                },
+                "values" : {
+                  "type" : "keyword",
+                  "ignore_above" : 256
+                }
+              }
+            },
+            "operator" : {
+              "type" : "keyword",
+              "ignore_above" : 256
+            }
+          }
+        },
+        "condition_result" : {
+          "type" : "object",
+          "enabled" : false
+        },
+        "context" : {
+          "type" : "keyword",
+          "copy_to" : [
+            "search_text"
+          ]
+        },
+        "created" : {
+          "type" : "date"
+        },
+        "expression" : {
+          "type" : "keyword",
+          "copy_to" : [
+            "search_text"
+          ]
+        },
+        "id" : {
+          "type" : "keyword"
+        },
+        "is_escalated" : {
+          "type" : "boolean"
+        },
+        "is_notified" : {
+          "type" : "boolean"
+        },
+        "message" : {
+          "type" : "keyword",
+          "ignore_above" : 256
+        },
+        "objects" : {
+          "type" : "keyword",
+          "copy_to" : [
+            "search_text"
+          ]
+        },
+        "resource_id" : {
+          "type" : "keyword"
+        },
+        "resource_name" : {
+          "type" : "keyword"
+        },
+        "rule_id" : {
+          "type" : "keyword"
+        },
+        "rule_name" : {
+          "type" : "keyword"
+        },
+        "search_text" : {
+          "type" : "text",
+          "analyzer" : "suggest_text_search",
+          "index_prefixes" : {
+            "min_chars" : 2,
+            "max_chars" : 5
+          },
+          "index_phrases" : true
+        },
+        "severity" : {
+          "type" : "keyword"
+        },
+        "state" : {
+          "type" : "keyword",
+          "ignore_above" : 256
+        },
+        "title" : {
+          "type" : "keyword"
+        },
+        "updated" : {
+          "type" : "date"
+        }
+      }
+    }
+}
+
+
+PUT _template/$[[INDEX_PREFIX]]activities-rollover
+{
+    "order" : 100000,
+    "index_patterns" : [
+      "$[[INDEX_PREFIX]]activities*"
+    ],
+    "settings" : {
+      "index" : {
+        "format" : "7",
+        "lifecycle" : {
+          "name" : "ilm_$[[INDEX_PREFIX]]metrics-30days-retention",
+          "rollover_alias" : "$[[INDEX_PREFIX]]activities"
+        },
+        "codec" : "best_compression",
+        "number_of_shards" : "1",
+        "translog.durability":"async"
+      }
+    },
+    "mappings" : {
+      "dynamic_templates" : [
+        {
+          "strings" : {
+            "mapping" : {
+              "ignore_above" : 256,
+              "type" : "keyword"
+            },
+            "match_mapping_type" : "string"
+          }
+        }
+      ]
+    },
+    "aliases" : { }
+  }
+
+
+PUT $[[INDEX_PREFIX]]activities-00001
+{
+"settings": {
+  "index.lifecycle.rollover_alias":"$[[INDEX_PREFIX]]activities"
+  , "refresh_interval": "5s"
+},
+"aliases":{
+  "$[[INDEX_PREFIX]]activities":{
+    "is_write_index":true
+  }
+}
+}
+
+

config/system_config.tpl

@@ -0,0 +1,76 @@
+
+elasticsearch:
+  - id: $[[CLUSTER_ID]]
+    name: $[[CLUSTER_ID]]
+    enabled: true
+    monitored: true
+    reserved: true
+    endpoint: $[[CLUSTER_ENDPINT]]
+    basic_auth:
+      username: $[[CLUSTER_USER]]
+      password: $[[CLUSTER_PASS]]
+
+elastic.elasticsearch: $[[CLUSTER_ID]]
+
+pipeline:
+  - name: indexing_merge
+    auto_start: true
+    keep_running: true
+    processor:
+      - indexing_merge:
+          input_queue: "metrics"
+          elasticsearch: "$[[CLUSTER_ID]]"
+          index_name: "$[[INDEX_PREFIX]]metrics"
+          output_queue:
+            name: "metrics_requests"
+            label:
+              tag: "metrics"
+          worker_size: 1
+          bulk_size_in_mb: 5
+  - name: consume-metrics_requests
+    auto_start: true
+    keep_running: true
+    processor:
+      - bulk_indexing:
+          bulk:
+            compress: true
+            batch_size_in_mb: 5
+            batch_size_in_docs: 5000
+          consumer:
+            fetch_max_messages: 100
+          queues:
+            type: indexing_merge
+          when:
+            cluster_available: ["$[[CLUSTER_ID]]"]
+  - name: metadata_ingest
+    auto_start: true
+    keep_running: true
+    processor:
+      - metadata:
+          bulk_size_in_mb: 5
+          bulk_max_docs_count: 5000
+          fetch_max_messages: 100
+          elasticsearch: "$[[CLUSTER_ID]]"
+          queues:
+            type: metadata
+            category: elasticsearch
+          consumer:
+            group: metadata
+          when:
+            cluster_available: ["$[[CLUSTER_ID]]"]
+  - name: activity_ingest
+    auto_start: true
+    keep_running: true
+    processor:
+      - activity:
+          bulk_size_in_mb: 5
+          bulk_max_docs_count: 5000
+          fetch_max_messages: 100
+          elasticsearch: "$[[CLUSTER_ID]]"
+          queues:
+            category: elasticsearch
+            activity: true
+          consumer:
+            group: activity
+          when:
+            cluster_available: ["$[[CLUSTER_ID]]"]

console.yml

@@ -1,20 +1,10 @@
-# for the system cluster, please use Elasticsearch v7.3+
+path.configs: "config"
-elasticsearch:
-  - name: default
-    enabled: true
-    monitored: false
-    endpoint: http://localhost:9200
-    basic_auth:
-      username: elastic
-      password: infinilabs
-    discovery:
-      enabled: true
 
 web:
   enabled: true
   embedding_api: true
   auth:
-    enabled: false
+    enabled: true
   ui:
     enabled: true
     path: .public
@@ -27,7 +17,6 @@ web:
     enabled: true
 
 elastic:
-  elasticsearch: default
   enabled: true
   remote_configs: true
   health_check:
@@ -59,66 +48,3 @@ metrics:
     cluster_stats: true
     node_stats: true
     index_stats: true
-
-pipeline:
-  - name: indexing_merge
-    auto_start: true
-    keep_running: true
-    processor:
-      - indexing_merge:
-          input_queue: "metrics"
-          elasticsearch: "default"
-          index_name: ".infini_metrics"
-          output_queue:
-            name: "metrics_requests"
-            label:
-              tag: "metrics"
-          worker_size: 1
-          bulk_size_in_mb: 10
-  - name: consume-metrics_requests
-    auto_start: true
-    keep_running: true
-    processor:
-      - bulk_indexing:
-          bulk:
-            compress: true
-            batch_size_in_mb: 10
-            batch_size_in_docs: 5000
-          consumer:
-            fetch_max_messages: 100
-          queues:
-            type: indexing_merge
-          when:
-            cluster_available: ["default"]
-  - name: metadata_ingest
-    auto_start: true
-    keep_running: true
-    processor:
-      - metadata:
-          bulk_size_in_mb: 10
-          bulk_max_docs_count: 5000
-          fetch_max_messages: 1000
-          elasticsearch: "default"
-          queues:
-            type: metadata
-            category: elasticsearch
-          consumer:
-            group: metadata
-          when:
-            cluster_available: ["default"]
-  - name: activity_ingest
-    auto_start: true
-    keep_running: true
-    processor:
-      - activity:
-          bulk_size_in_mb: 10
-          bulk_max_docs_count: 5000
-          fetch_max_messages: 1000
-          elasticsearch: "default"
-          queues:
-            category: elasticsearch
-            activity: true
-          consumer:
-            group: activity
-          when:
-            cluster_available: ["default"]

main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"errors"
 	_ "expvar"
 	log "github.com/cihub/seelog"
@@ -12,10 +13,12 @@ import (
 	"infini.sh/framework"
 	"infini.sh/framework/core/elastic"
 	"infini.sh/framework/core/env"
+	"infini.sh/framework/core/global"
 	"infini.sh/framework/core/insight"
 	_ "infini.sh/framework/core/log"
 	"infini.sh/framework/core/module"
 	"infini.sh/framework/core/orm"
+	task1 "infini.sh/framework/core/task"
 	"infini.sh/framework/modules/agent"
 	_ "infini.sh/framework/modules/api"
 	elastic2 "infini.sh/framework/modules/elastic"
@@ -23,10 +26,13 @@ import (
 	"infini.sh/framework/modules/pipeline"
 	queue2 "infini.sh/framework/modules/queue/disk_queue"
 	"infini.sh/framework/modules/redis"
+	"infini.sh/framework/modules/security"
 	"infini.sh/framework/modules/stats"
 	"infini.sh/framework/modules/task"
 	"infini.sh/framework/modules/ui"
 	_ "infini.sh/framework/plugins"
+	setup1 "infini.sh/console/plugin/setup"
+	_ "infini.sh/console/plugin"
 	api2 "infini.sh/gateway/api"
 	_ "infini.sh/gateway/proxy"
 	_ "time/tzdata"
@@ -36,12 +42,7 @@ var appConfig *config.AppConfig
 var appUI *UI
 
 func main() {
-	terminalHeader := ("\n\n")
+	terminalHeader := ("\n")
-	terminalHeader += ("   ___  __   ___         ___          \n")
-	terminalHeader += ("  / __\\/ /  /___\\/\\ /\\  /   \\         \n")
-	terminalHeader += (" / /  / /  //  // / \\ \\/ /\\ /         \n")
-	terminalHeader += ("/ /__/ /__/ \\_//\\ \\_/ / /_//          \n")
-	terminalHeader += ("\\____|____|___/  \\___/___,'           \n")
 	terminalHeader += ("   ___  ___    __  __    ___  __   __ \n")
 	terminalHeader += ("  / __\\/___\\/\\ \\ \\/ _\\  /___\\/ /  /__\\\n")
 	terminalHeader += (" / /  //  //  \\/ /\\ \\  //  // /  /_\\  \n")
@@ -51,7 +52,7 @@ func main() {
 
 	terminalFooter := ""
 
-	app := framework.NewApp("console", "INFINI Cloud Console, The easiest way to operate your own elasticsearch platform.",
+	app := framework.NewApp("console", "The easiest way to operate your own search platform.",
 		config.Version, config.BuildNumber, config.LastCommitLog, config.BuildDate, config.EOLDate, terminalHeader, terminalFooter)
 
 	app.Init(nil)
@@ -59,28 +60,49 @@ func main() {
 
 	api := api2.GatewayAPI{}
 
+	modules:=[]module.Module{}
+	modules=append(modules,&stats.SimpleStatsModule{})
+	modules=append(modules,&elastic2.ElasticModule{})
+	modules=append(modules,&queue2.DiskQueue{})
+	modules=append(modules,&redis.RedisModule{})
+	modules=append(modules,&pipeline.PipeModule{})
+	modules=append(modules,&task.TaskModule{})
+	modules=append(modules,&agent.AgentModule{})
+	modules=append(modules,&metrics.MetricsModule{})
+	modules=append(modules,&security.Module{})
+
+	uiModule:=&ui.UIModule{}
+
 	if app.Setup(func() {
-		err := bootstrapRequirementCheck()
-		if err != nil {
-			panic(err)
-		}
 
 		//load core modules first
-		module.RegisterSystemModule(&elastic2.ElasticModule{})
+		module.RegisterSystemModule(&setup1.Module{})
+		module.RegisterSystemModule(uiModule)
+
+		var initFunc= func() {
 			module.RegisterSystemModule(&stats.SimpleStatsModule{})
+			module.RegisterSystemModule(&elastic2.ElasticModule{})
 			module.RegisterSystemModule(&queue2.DiskQueue{})
 			module.RegisterSystemModule(&redis.RedisModule{})
-		module.RegisterSystemModule(&ui.UIModule{})
 			module.RegisterSystemModule(&pipeline.PipeModule{})
 			module.RegisterSystemModule(&task.TaskModule{})
 			module.RegisterSystemModule(&agent.AgentModule{})
+			module.RegisterSystemModule(&metrics.MetricsModule{})
+			module.RegisterSystemModule(&security.Module{})
+		}
 
-		module.RegisterUserPlugin(&metrics.MetricsModule{})
+		if !global.Env().SetupRequired(){
+			initFunc()
+		}else{
+			for _, v := range modules {
+				v.Setup()
+			}
+			setup1.RegisterSetupCallback(initFunc)
+		}
 
 		api.RegisterAPI("")
 
 		appConfig = &config.AppConfig{
-			Elasticsearch: "default",
 			UI: config.UIConfig{
 				LocalPath:    ".public",
 				VFSEnabled:   true,
@@ -104,8 +126,16 @@ func main() {
 
 		module.Start()
 
+		var initFunc= func() {
+			if global.Env().SetupRequired() {
+				for _, v := range modules {
+					v.Start()
+				}
+			}
+
+			elastic2.InitTemplate(false)
+
 			//orm.RegisterSchemaWithIndexName(model.Dict{}, "dict")
-		//orm.RegisterSchemaWithIndexName(model.Reindex{}, "reindex")
 			orm.RegisterSchemaWithIndexName(elastic.View{}, "view")
 			orm.RegisterSchemaWithIndexName(elastic.CommonCommand{}, "commands")
 			//orm.RegisterSchemaWithIndexName(elastic.TraceTemplate{}, "trace-template")
@@ -118,12 +148,27 @@ func main() {
 			orm.RegisterSchemaWithIndexName(insight.Dashboard{}, "dashboard")
 			api.RegisterSchema()
 
-		go func() {
+			task1.RunWithinGroup("initialize_alerting",func(ctx context.Context) error {
 				err := alerting2.InitTasks()
 				if err != nil {
 					log.Errorf("init alerting task error: %v", err)
 				}
-		}()
+				return err
+			})
+		}
+
+		if !global.Env().SetupRequired(){
+			initFunc()
+		}else{
+			setup1.RegisterSetupCallback(initFunc)
+		}
+
+		if !global.Env().SetupRequired(){
+			err := bootstrapRequirementCheck()
+			if err != nil {
+				panic(err)
+			}
+		}
 
 	}, nil) {
 		app.Run()

plugin/api/alerting/alert.go

@@ -10,6 +10,7 @@ import (
 	"infini.sh/console/model/alerting"
 	httprouter "infini.sh/framework/core/api/router"
 	"infini.sh/framework/core/elastic"
+	"infini.sh/framework/core/global"
 	"infini.sh/framework/core/orm"
 	"infini.sh/framework/core/util"
 	"net/http"
@@ -109,7 +110,7 @@ func (h *AlertAPI) searchAlert(w http.ResponseWriter, req *http.Request, ps http
 }
 
 func (h *AlertAPI) getAlertStats(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
-	esClient := elastic.GetClient(h.Config.Elasticsearch)
+	esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
 	queryDsl := util.MapStr{
 		"size": 0,
 		"query": util.MapStr{

plugin/api/alerting/api.go

@@ -5,7 +5,6 @@
 package alerting
 
 import (
-	"infini.sh/console/config"
 	"infini.sh/framework/core/api"
 	"infini.sh/framework/core/api/rbac/enum"
 )
@@ -13,7 +12,6 @@ import (
 
 type AlertAPI struct {
 	api.Handler
-	Config *config.AppConfig
 }
 
 func (alert *AlertAPI) Init() {

plugin/api/alerting/message.go

@@ -11,6 +11,7 @@ import (
 	alerting2 "infini.sh/console/service/alerting"
 	httprouter "infini.sh/framework/core/api/router"
 	"infini.sh/framework/core/elastic"
+	"infini.sh/framework/core/global"
 	"infini.sh/framework/core/kv"
 	"infini.sh/framework/core/orm"
 	"infini.sh/framework/core/util"
@@ -82,7 +83,7 @@ func (h *AlertAPI) ignoreAlertMessage(w http.ResponseWriter, req *http.Request,
 }
 
 func (h *AlertAPI) getAlertMessageStats(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
-	esClient := elastic.GetClient(h.Config.Elasticsearch)
+	esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
 	must := []util.MapStr{
 		{
 			"terms": util.MapStr{

plugin/api/alerting/rule.go

@@ -14,6 +14,7 @@ import (
 	httprouter "infini.sh/framework/core/api/router"
 	"infini.sh/framework/core/elastic"
 	"infini.sh/framework/core/event"
+	"infini.sh/framework/core/global"
 	"infini.sh/framework/core/kv"
 	"infini.sh/framework/core/orm"
 	"infini.sh/framework/core/queue"
@@ -462,7 +463,8 @@ func (alertAPI *AlertAPI) searchRule(w http.ResponseWriter, req *http.Request, p
 }
 
 func (alertAPI *AlertAPI) getRuleAlertMessageNumbers(ruleIDs []string) ( map[string]interface{},error) {
-	esClient := elastic.GetClient(alertAPI.Config.Elasticsearch)
+
+	esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
 	queryDsl := util.MapStr{
 		"size": 0,
 		"query": util.MapStr{
@@ -513,7 +515,7 @@ func (alertAPI *AlertAPI) fetchAlertInfos(w http.ResponseWriter, req *http.Reque
 		alertAPI.WriteJSON(w, util.MapStr{}, http.StatusOK)
 		return
 	}
-	esClient := elastic.GetClient(alertAPI.Config.Elasticsearch)
+	esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
 	queryDsl := util.MapStr{
 		"_source": []string{"state", "rule_id"},
 		"sort": []util.MapStr{

plugin/api/gateway/api.go

@@ -13,7 +13,7 @@ type GatewayAPI struct {
 	api.Handler
 }
 
-func init() {
+func InitAPI() {
 	gateway:=GatewayAPI{}
 	api.HandleAPIMethod(api.POST, "/gateway/instance/try_connect", gateway.RequireLogin(gateway.tryConnect))
 	api.HandleAPIMethod(api.GET, "/gateway/instance/:instance_id", gateway.RequirePermission(gateway.getInstance, enum.PermissionGatewayInstanceRead))

plugin/api/index_management/common_command.go

@@ -5,6 +5,7 @@ import (
 	log "github.com/cihub/seelog"
 	httprouter "infini.sh/framework/core/api/router"
 	"infini.sh/framework/core/elastic"
+	"infini.sh/framework/core/global"
 	"infini.sh/framework/core/orm"
 	"infini.sh/framework/core/util"
 	"net/http"
@@ -28,7 +29,7 @@ func (h *APIHandler) HandleAddCommonCommandAction(w http.ResponseWriter, req *ht
 
 	reqParams.Created = time.Now()
 	reqParams.ID = util.GetUUID()
-	esClient := elastic.GetClient(h.Config.Elasticsearch)
+	esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
 
 	queryDSL :=[]byte(fmt.Sprintf(`{"size":1, "query":{"bool":{"must":{"match":{"title.keyword":"%s"}}}}}`, reqParams.Title))
 	var indexName  = orm.GetIndexName(reqParams)
@@ -73,7 +74,7 @@ func (h *APIHandler) HandleSaveCommonCommandAction(w http.ResponseWriter, req *h
 		return
 	}
 	reqParams.ID = ps.ByName("cid")
-	esClient := elastic.GetClient(h.Config.Elasticsearch)
+	esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
 
 	queryDSL :=[]byte(fmt.Sprintf(`{"size":1, "query":{"bool":{"must":{"match":{"title.keyword":"%s"}}}}}`, reqParams.Title))
 	var indexName  = orm.GetIndexName(reqParams)
@@ -133,7 +134,7 @@ func (h *APIHandler) HandleQueryCommonCommandAction(w http.ResponseWriter, req *
 	}
 
 	queryDSL = fmt.Sprintf(queryDSL, filterBuilder.String(), size, from)
-	esClient := elastic.GetClient(h.Config.Elasticsearch)
+	esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
 
 	searchRes, err := esClient.SearchWithRawQueryDSL(orm.GetIndexName(elastic.CommonCommand{}), []byte(queryDSL))
 	if err != nil {
@@ -149,7 +150,7 @@ func (h *APIHandler) HandleQueryCommonCommandAction(w http.ResponseWriter, req *
 func (h *APIHandler) HandleDeleteCommonCommandAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
 	resBody := map[string]interface{}{}
 	id := ps.ByName("cid")
-	esClient := elastic.GetClient(h.Config.Elasticsearch)
+	esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
 	delRes, err := esClient.Delete(orm.GetIndexName(elastic.CommonCommand{}), "", id, "wait_for")
 	if err != nil {
 		log.Error(err)

plugin/api/index_management/elasticsearch.go

@@ -6,6 +6,7 @@ import (
 	httprouter "infini.sh/framework/core/api/router"
 	"infini.sh/framework/core/elastic"
 	"infini.sh/framework/core/event"
+	"infini.sh/framework/core/global"
 	"infini.sh/framework/core/host"
 	"infini.sh/framework/core/orm"
 	"infini.sh/framework/core/util"
@@ -25,7 +26,7 @@ func (handler APIHandler) ElasticsearchOverviewAction(w http.ResponseWriter, req
 	//	clusterIDs = append(clusterIDs, key)
 	//	return true
 	//})
-	esClient := elastic.GetClient(handler.Config.Elasticsearch)
+	esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
 	queryDsl := util.MapStr{
 		"size": 100,
 	}
@@ -110,7 +111,7 @@ func (handler APIHandler) ElasticsearchOverviewAction(w http.ResponseWriter, req
 }
 
 func (handler APIHandler) getLatestClusterMonitorData(clusterIDs []interface{}) (*elastic.SearchResponse, error){
-	client := elastic.GetClient(handler.Config.Elasticsearch)
+	client := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
 	queryDSLTpl := `{
   "size": %d, 
    "query": {
@@ -155,7 +156,7 @@ func (handler APIHandler) getLatestClusterMonitorData(clusterIDs []interface{})
 }
 
 func (handler APIHandler) getMetricCount(indexName, field string, clusterIDs []interface{}) (interface{}, error){
-	client := elastic.GetClient(handler.Config.Elasticsearch)
+	client := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
 	queryDSL := util.MapStr{
   "size": 0, 
   "aggs": util.MapStr{
@@ -182,7 +183,7 @@ func (handler APIHandler) getMetricCount(indexName, field string, clusterIDs []i
 }
 
 func (handler APIHandler) getLastActiveHostCount() (int, error){
-	client := elastic.GetClient(handler.Config.Elasticsearch)
+	client := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
 	queryDSL := `{
   "size": 0, 
   "query": {

plugin/api/index_management/index.go

@@ -1,6 +1,8 @@
 package index_management
 
 import (
+	"infini.sh/framework/core/elastic"
+	"infini.sh/framework/core/global"
 	"net/http"
 	"strconv"
 	"strings"
@@ -33,7 +35,7 @@ func (handler APIHandler) GetDictListAction(w http.ResponseWriter, req *http.Req
 	if len(tags) > 3 {
 		tags = tags[0:3]
 	}
-	rel, err := model2.GetDictList(from, size, name, tags, handler.Config.Elasticsearch)
+	rel, err := model2.GetDictList(from, size, name, tags, global.MustLookupString(elastic.GlobalSystemElasticsearchID))
 	if err != nil {
 		resp["error"] = err
 		resp["status"] = false

plugin/api/index_management/rebuild.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	log "github.com/cihub/seelog"
+	"infini.sh/framework/core/global"
 	"infini.sh/framework/core/orm"
 	"net/http"
 	"strings"
@@ -33,7 +34,7 @@ func (handler APIHandler) HandleReindexAction(w http.ResponseWriter, req *http.R
 
 	//fmt.Println(reindexItem)
 	typ := handler.GetParameter(req, "_type")
-	ID, err := reindex(handler.Config.Elasticsearch, reindexItem, typ)
+	ID, err := reindex(global.MustLookupString(elastic.GlobalSystemElasticsearchID), reindexItem, typ)
 	if err != nil {
 		log.Error(err)
 		resResult["error"] = err
@@ -94,7 +95,7 @@ func (handler APIHandler) HandleDeleteRebuildAction(w http.ResponseWriter, req *
 	id := ps.ByName("id")
 	var ids = []string{id}
 	resBody := newResponseBody()
-	err := deleteTasksByIds(handler.Config.Elasticsearch, ids)
+	err := deleteTasksByIds(global.MustLookupString(elastic.GlobalSystemElasticsearchID), ids)
 	if err != nil {
 		log.Error(err)
 		resBody["error"] = err
@@ -111,7 +112,7 @@ func (handler APIHandler) HandleGetRebuildListAction(w http.ResponseWriter, req
 		size    = handler.GetIntOrDefault(req, "size", 10)
 		name    = handler.GetParameter(req, "name")
 		resBody = newResponseBody()
-		esName  = handler.Config.Elasticsearch
+		esName  = global.MustLookupString(elastic.GlobalSystemElasticsearchID)
 	)
 	esResp, err := model.GetRebuildList(esName, from, size, name)
 	if err != nil {

plugin/api/init.go

@@ -3,7 +3,9 @@ package api
 import (
 	"infini.sh/console/config"
 	"infini.sh/console/plugin/api/alerting"
+	"infini.sh/console/plugin/api/gateway"
 	"infini.sh/console/plugin/api/index_management"
+	"infini.sh/console/plugin/api/insight"
 	"infini.sh/framework/core/api"
 	"infini.sh/framework/core/api/rbac/enum"
 	"path"
@@ -58,9 +60,10 @@ func Init(cfg *config.AppConfig) {
 	//})
 
 	alertAPI := alerting.AlertAPI{
-		Config: cfg,
 	}
 
 	alertAPI.Init()
 
+	gateway.InitAPI()
+	insight.InitAPI()
 }

plugin/api/insight/api.go

@@ -10,7 +10,7 @@ type InsightAPI struct {
 	api.Handler
 }
 
-func init() {
+func InitAPI() {
 	insight := InsightAPI{}
 	api.HandleAPIMethod(api.POST, "/elasticsearch/:id/visualization/metadata", insight.HandleGetMetadata)
 	api.HandleAPIMethod(api.POST, "/elasticsearch/:id/visualization/data", insight.HandleGetMetricData)

plugin/setup/setup.go

@@ -0,0 +1,437 @@
+package task
+
+import (
+	"bytes"
+	"fmt"
+	"golang.org/x/crypto/bcrypt"
+	"infini.sh/framework/core/api"
+	"infini.sh/framework/core/api/rbac"
+	httprouter "infini.sh/framework/core/api/router"
+	"infini.sh/framework/core/elastic"
+	"infini.sh/framework/core/env"
+	"infini.sh/framework/core/errors"
+	"infini.sh/framework/core/global"
+	"infini.sh/framework/core/module"
+	"infini.sh/framework/core/orm"
+	"infini.sh/framework/core/util"
+	elastic2 "infini.sh/framework/modules/elastic"
+	elastic1 "infini.sh/framework/modules/elastic/common"
+	elastic3 "infini.sh/framework/modules/elastic/api"
+	"infini.sh/framework/modules/security"
+	"io"
+	"net/http"
+	uri2 "net/url"
+	"path"
+	"runtime"
+	"github.com/valyala/fasttemplate"
+	"time"
+)
+
+type Module struct {
+	api.Handler
+}
+
+func (module *Module) Name() string {
+	return "setup"
+}
+
+func init()  {
+	module.RegisterSystemModule(&Module{})
+}
+
+func (module *Module) Setup() {
+
+	if !global.Env().SetupRequired(){
+		return
+	}
+
+	api.HandleAPIMethod(api.POST, "/setup/_validate", module.validate)
+	api.HandleAPIMethod(api.POST, "/setup/_initialize", module.initialize)
+	elastic3.InitTestAPI()
+}
+
+var setupFinishedCallback= []func() {}
+func RegisterSetupCallback(f func()) {
+	setupFinishedCallback=append(setupFinishedCallback,f)
+}
+
+func InvokeSetupCallback()  {
+	for _,v:=range setupFinishedCallback{
+		v()
+	}
+}
+
+func (module *Module) Start() error {
+	return nil
+}
+func (module *Module) Stop() error {
+	return nil
+}
+
+type SetupRequest struct {
+	Cluster struct {
+		Host string `json:"host"`
+		Schema string `json:"schema"`
+		Endpoint string `json:"endpoint"`
+		Username string `json:"username"`
+		Password string `json:"password"`
+	} `json:"cluster"`
+
+	BootstrapUsername string `json:"bootstrap_username"`
+	BootstrapPassword string `json:"bootstrap_password"`
+}
+
+var tempID="infini_system_cluster_"+util.GetUUID()
+
+const VersionTooOld ="elasticsearch_version_too_old"
+const IndicesExists ="elasticsearch_indices_exists"
+const TemplateExists ="elasticsearch_template_exists"
+
+var cfg1 elastic1.ORMConfig
+
+func (module *Module) validate(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+
+	if !global.Env().SetupRequired(){
+		module.WriteError(w, "setup not permitted", 500)
+		return
+	}
+
+	success:=false
+	var err error
+	var errType string
+	var fixTips string
+	var code int
+	code=200
+	defer func() {
+
+		global.Env().CheckSetup()
+
+		result := util.MapStr{}
+		result["success"]=success
+
+		if r := recover(); r != nil {
+			var v string
+			switch r.(type) {
+			case error:
+				v = r.(error).Error()
+			case runtime.Error:
+				v = r.(runtime.Error).Error()
+			case string:
+				v = r.(string)
+			}
+			if v!=""{
+				success=false
+				result["error"]=util.MapStr{
+					"reason":v,
+				}
+				if errType!=""{
+					result["type"]=errType
+				}
+				if fixTips!=""{
+					result["fix_tips"]=fixTips
+				}
+				code=500
+			}
+		}
+		module.WriteJSON(w, result, code)
+	}()
+
+
+	err, client,_ := module.initTempClient(r)
+	if err!=nil{
+		panic(err)
+	}
+
+	//validate version
+	version := client.GetVersion()
+	if version != "" {
+		ver := &util.Version{}
+		ver, err = util.ParseSemantic(version)
+		if err != nil {
+			panic(err)
+		}
+		if ver.Major() >= 7 {
+			if ver.Major() == 7 && ver.Minor() < 3 {
+				errType = VersionTooOld
+				panic(errors.Errorf("elasticsearch version(%v) should greater than v7.3", version))
+			}
+		} else {
+			errType = VersionTooOld
+			panic(errors.Errorf("elasticsearch version(%v) should greater than v7.3", version))
+		}
+	}
+	cfg1 = elastic1.ORMConfig{}
+	exist, err := env.ParseConfig("elastic.orm", &cfg1)
+	if exist && err != nil {
+		panic(err)
+	}
+
+	if cfg1.IndexPrefix==""{
+		cfg1.IndexPrefix=".infini_"
+	}
+	if cfg1.TemplateName==""{
+		cfg1.TemplateName=".infini"
+	}
+
+	//validate indices
+	indices, err := client.GetIndices(util.TrimSpaces(cfg1.IndexPrefix) + "*")
+	if err != nil {
+		panic(err)
+	}
+
+	if indices != nil && len(*indices) > 0 {
+		buff := bytes.Buffer{}
+		tipBuff := bytes.Buffer{}
+		for k, _ := range *indices {
+			buff.WriteString(k)
+			buff.WriteString("\n")
+
+			tipBuff.WriteString("DELETE ")
+			tipBuff.WriteString(k)
+			tipBuff.WriteString("\n")
+		}
+		errType = IndicesExists
+		fixTips=tipBuff.String()
+		panic(errors.Errorf("there are following indices exists in target elasticsearch: \n%v", buff.String()))
+	}
+
+	ok, err := client.TemplateExists(cfg1.TemplateName)
+	if err != nil {
+		panic(err)
+	}
+	if ok {
+		errType = TemplateExists
+		fixTips="DELETE /_template/"+util.TrimSpaces(cfg1.TemplateName)
+		panic(errors.Errorf("there are following template already exists in target elasticsearch: \n%v", cfg1.TemplateName))
+	}
+
+	success = true
+}
+var cfg elastic.ElasticsearchConfig
+func (module *Module) initTempClient(r *http.Request) (error, elastic.API,SetupRequest) {
+	request := SetupRequest{}
+	err := module.DecodeJSON(r, &request)
+	if err != nil {
+		return err,nil,request
+	}
+
+	if request.Cluster.Endpoint==""&&request.Cluster.Host==""{
+		panic("invalid configuration")
+	}
+
+	if request.Cluster.Endpoint==""{
+		if request.Cluster.Host!=""&&request.Cluster.Schema!=""{
+			request.Cluster.Endpoint=fmt.Sprintf("%v://%v",request.Cluster.Schema,request.Cluster.Host)
+		}
+	}
+
+	cfg = elastic.ElasticsearchConfig{
+		Enabled:  true,
+		Reserved: true,
+		Endpoint: request.Cluster.Endpoint,
+		BasicAuth: &elastic.BasicAuth{
+			Username: request.Cluster.Username,
+			Password: request.Cluster.Password,
+		},
+	}
+
+	if cfg.Endpoint!=""&&cfg.Host==""{
+		uri,err:=uri2.Parse(cfg.Endpoint)
+		if err!=nil{
+			panic(err)
+		}
+		cfg.Host=uri.Host
+		cfg.Schema=uri.Scheme
+	}
+
+	cfg.ID = tempID
+	cfg.Name = "INFINI_SYSTEM ("+util.PickRandomName()+")"
+	elastic.InitMetadata(&cfg, true)
+	client, err := elastic1.InitClientWithConfig(cfg)
+	if err != nil {
+		return err,nil,request
+	}
+
+	elastic.UpdateConfig(cfg)
+	elastic.UpdateClient(cfg, client)
+
+	global.Register(elastic.GlobalSystemElasticsearchID,tempID)
+
+	return err, client,request
+}
+
+func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+	if !global.Env().SetupRequired(){
+		module.WriteError(w, "setup not permitted", 500)
+		return
+	}
+
+	success:=false
+	var err error
+	var errType string
+	var fixTips string
+	var code int
+	code=200
+	defer func() {
+
+		global.Env().CheckSetup()
+
+		result := util.MapStr{}
+		result["success"]=success
+
+		if r := recover(); r != nil {
+			var v string
+			switch r.(type) {
+			case error:
+				v = r.(error).Error()
+			case runtime.Error:
+				v = r.(runtime.Error).Error()
+			case string:
+				v = r.(string)
+			}
+			if v!=""{
+				success=false
+				result["error"]=util.MapStr{
+					"reason":v,
+				}
+				if errType!=""{
+					result["type"]=errType
+				}
+				if fixTips!=""{
+					result["fix_tips"]=fixTips
+				}
+				code=500
+			}
+		}
+		module.WriteJSON(w, result, code)
+	}()
+
+	err, client,request := module.initTempClient(r)
+	if err!=nil{
+		panic(err)
+	}
+
+	if cfg1.IndexPrefix==""{
+		cfg1.IndexPrefix=".infini_"
+	}
+	if cfg1.TemplateName==""{
+		cfg1.TemplateName=".infini"
+	}
+
+	if !cfg1.Enabled{
+		cfg1.Enabled=true
+	}
+
+	if !cfg1.InitTemplate{
+		cfg1.InitTemplate=true
+	}
+
+	cfg.Reserved=true
+	cfg.Monitored=true
+
+	//处理ORM
+	handler := elastic2.ElasticORM{Client: client, Config:cfg1 }
+	orm.Register("elastic_setup_"+util.GetUUID(), handler)
+
+	//处理模版
+	elastic2.InitTemplate(true)
+
+	//处理生命周期
+	//TEMPLATE_NAME
+	//INDEX_PREFIX
+
+	dslTplFile:=path.Join(global.Env().GetConfigDir(),"initialization.tpl")
+	dslFile:=path.Join(global.Env().GetConfigDir(),"initialization.dsl")
+
+	var dsl []byte
+	dsl,err=util.FileGetContent(dslTplFile)
+	if err!=nil{
+		panic(err)
+	}
+
+	if len(dsl)>0{
+		var tpl *fasttemplate.Template
+		tpl,err=fasttemplate.NewTemplate(string(dsl), "$[[", "]]")
+		if err!=nil{
+			panic(err)
+		}
+		if tpl!=nil{
+			output:=tpl.ExecuteFuncString(func(w io.Writer, tag string) (int, error) {
+				switch tag {
+				case "TEMPLATE_NAME":
+					return w.Write([]byte(cfg1.TemplateName))
+				case "INDEX_PREFIX":
+					return w.Write([]byte(cfg1.IndexPrefix))
+				}
+				panic(errors.Errorf("unknown tag: %v",tag))
+			})
+			_,err=util.FilePutContent(dslFile,output)
+			if err!=nil{
+				panic(err)
+			}
+		}
+	}
+
+
+	//处理索引
+	elastic2.InitSchema()
+	//init security
+	security.InitSecurity()
+
+	//保存默认集群
+	err=orm.Save(&cfg)
+	if err!=nil{
+		panic(err)
+	}
+
+	if request.BootstrapUsername!=""&&request.BootstrapPassword!=""{
+		//Save bootstrap user
+		user:=rbac.User{}
+		user.ID="default_user_"+request.BootstrapUsername
+		user.Name=request.BootstrapUsername
+		user.NickName=request.BootstrapUsername
+		var hash []byte
+		hash, err = bcrypt.GenerateFromPassword([]byte(request.BootstrapPassword), bcrypt.DefaultCost)
+		if err!=nil{
+			panic(err)
+		}
+
+		user.Password=string(hash)
+		role:=[]rbac.UserRole{}
+		role=append(role,rbac.UserRole{
+			ID: rbac.RoleAdminName,
+			Name: rbac.RoleAdminName,
+		})
+		user.Roles=role
+		err=orm.Save(&user)
+		if err!=nil{
+			panic(err)
+		}
+	}
+
+
+	//save to local file
+	file:=path.Join(global.Env().GetConfigDir(),"system_config.yml")
+	_,err=util.FilePutContent(file,fmt.Sprintf("configs.template:\n  - name: \"system\"\n    path: ./config/system_config.tpl\n    variable:\n      " +
+		"CLUSTER_ID: %v\n      CLUSTER_ENDPINT: \"%v\"\n      " +
+		"CLUSTER_USER: \"%v\"\n      CLUSTER_PASS: \"%v\"\n      INDEX_PREFIX: \"%v\"",
+	tempID,cfg.Endpoint,cfg.BasicAuth.Username,cfg.BasicAuth.Password,cfg1.IndexPrefix	))
+	if err!=nil{
+		panic(err)
+	}
+
+	//处理 ILM
+
+	//callback
+	InvokeSetupCallback()
+
+	//place setup lock file
+	setupLock:=path.Join(global.Env().GetDataDir(),".setup_lock")
+	_,err=util.FilePutContent(setupLock,time.Now().String())
+	if err!=nil{
+		panic(err)
+	}
+
+	success=true
+
+}