improve setup, fix auth issue

config/config.go

@@ -3,7 +3,6 @@ package config
 import "infini.sh/framework/core/config"
 
 type AppConfig struct {
-	Elasticsearch string `config:"elasticsearch"`
 	UI UIConfig `config:"ui"`
 	Network config.NetworkConfig `config:"network"`
 	TLSConfig   config.TLSConfig     `config:"tls"`

config/initialization.tpl

@@ -1,8 +1,8 @@
-PUT _template/.infini
+PUT _template/$[[TEMPLATE_NAME]]
 {
     "order": 0,
     "index_patterns": [
-      ".infini_*"
+      "$[[INDEX_PREFIX]]*"
     ],
     "settings": {
       "index": {
@@ -41,7 +41,7 @@ PUT _template/.infini
     "aliases": {}
 }
 
-PUT _ilm/policy/infini_metrics-30days-retention
+PUT _ilm/policy/ilm_$[[INDEX_PREFIX]]metrics-30days-retention
 {
   "policy": {
     "phases": {
@@ -69,18 +69,18 @@ PUT _ilm/policy/infini_metrics-30days-retention
   }
 }
 
-PUT _template/.infini_metrics-rollover
+PUT _template/$[[INDEX_PREFIX]]metrics-rollover
 {
     "order" : 100000,
     "index_patterns" : [
-      ".infini_metrics*"
+      "$[[INDEX_PREFIX]]metrics*"
     ],
     "settings" : {
       "index" : {
         "format" : "7",
         "lifecycle" : {
-          "name" : "infini_metrics-30days-retention",
+          "name" : "ilm_$[[INDEX_PREFIX]]metrics-30days-retention",
-          "rollover_alias" : ".infini_metrics"
+          "rollover_alias" : "$[[INDEX_PREFIX]]metrics"
         },
         "codec" : "best_compression",
         "number_of_shards" : "1",
@@ -104,16 +104,14 @@ PUT _template/.infini_metrics-rollover
   }
 
 
-# DELETE .infini_metrics
+PUT $[[INDEX_PREFIX]]metrics-00001
-# DELETE .infini_metrics-00001
-PUT .infini_metrics-00001
 {
   "settings": {
-    "index.lifecycle.rollover_alias":".infini_metrics"
+    "index.lifecycle.rollover_alias":"$[[INDEX_PREFIX]]metrics"
     , "refresh_interval": "5s"
   },
   "aliases":{
-    ".infini_metrics":{
+    "$[[INDEX_PREFIX]]metrics":{
       "is_write_index":true
     }
   }
@@ -121,18 +119,18 @@ PUT .infini_metrics-00001
 
 
 
-PUT _template/.infini_alert-history-rollover
+PUT _template/$[[INDEX_PREFIX]]alert-history-rollover
 {
     "order" : 100000,
     "index_patterns" : [
-      ".infini_alert-history*"
+      "$[[INDEX_PREFIX]]alert-history*"
     ],
     "settings" : {
       "index" : {
         "format" : "7",
         "lifecycle" : {
-          "name" : "infini_metrics-30days-retention",
+          "name" : "ilm_$[[INDEX_PREFIX]]metrics-30days-retention",
-          "rollover_alias" : ".infini_alert-history"
+          "rollover_alias" : "$[[INDEX_PREFIX]]alert-history"
         },
         "codec" : "best_compression",
         "number_of_shards" : "1",
@@ -155,16 +153,15 @@ PUT _template/.infini_alert-history-rollover
     "aliases" : { }
   }
 
-# DELETE .infini_alert-history
+
-# DELETE .infini_alert-history-00001
+PUT $[[INDEX_PREFIX]]alert-history-00001
-PUT .infini_alert-history-00001
 {
   "settings": {
-    "index.lifecycle.rollover_alias":".infini_alert-history"
+    "index.lifecycle.rollover_alias":"$[[INDEX_PREFIX]]alert-history"
     , "refresh_interval": "5s"
   },
   "aliases":{
-    ".infini_alert-history":{
+    "$[[INDEX_PREFIX]]alert-history":{
       "is_write_index":true
     }
   },
@@ -278,20 +275,18 @@ PUT .infini_alert-history-00001
 }
 
 
-
+PUT _template/$[[INDEX_PREFIX]]activities-rollover
-
-PUT _template/.infini_activities-rollover
 {
     "order" : 100000,
     "index_patterns" : [
-      ".infini_activities*"
+      "$[[INDEX_PREFIX]]activities*"
     ],
     "settings" : {
       "index" : {
         "format" : "7",
         "lifecycle" : {
-          "name" : "infini_metrics-30days-retention",
+          "name" : "ilm_$[[INDEX_PREFIX]]metrics-30days-retention",
-          "rollover_alias" : ".infini_activities"
+          "rollover_alias" : "$[[INDEX_PREFIX]]activities"
         },
         "codec" : "best_compression",
         "number_of_shards" : "1",
@@ -314,17 +309,15 @@ PUT _template/.infini_activities-rollover
     "aliases" : { }
   }
 
-#DELETE .infini_activities
-#DELETE .infini_activities-00001
 
-PUT .infini_activities-00001
+PUT $[[INDEX_PREFIX]]activities-00001
 {
 "settings": {
-  "index.lifecycle.rollover_alias":".infini_activities"
+  "index.lifecycle.rollover_alias":"$[[INDEX_PREFIX]]activities"
   , "refresh_interval": "5s"
 },
 "aliases":{
-  ".infini_activities":{
+  "$[[INDEX_PREFIX]]activities":{
     "is_write_index":true
   }
 }

config/system_config.tpl

@@ -3,7 +3,7 @@ elasticsearch:
   - id: $[[CLUSTER_ID]]
     name: $[[CLUSTER_ID]]
     enabled: true
-    monitored: false
+    monitored: true
     reserved: true
     endpoint: $[[CLUSTER_ENDPINT]]
     basic_auth:

config/system_config.yml

@@ -1,9 +0,0 @@
-configs.template:
-  - name: "system"
-    path: ./config/system_config.tpl
-    variable:
-      CLUSTER_ID: _setup_clustercd9s5brq50k75kji57tg
-      CLUSTER_ENDPINT: "http://192.168.3.188:9206"
-      CLUSTER_USER: "admin"
-      CLUSTER_PASS: "pass"
-      INDEX_PREFIX: ".infini_"

main.go

@@ -69,13 +69,15 @@ func main() {
 	modules=append(modules,&task.TaskModule{})
 	modules=append(modules,&agent.AgentModule{})
 	modules=append(modules,&metrics.MetricsModule{})
+	modules=append(modules,&security.Module{})
 
+	uiModule:=&ui.UIModule{}
 
 	if app.Setup(func() {
 
 		//load core modules first
 		module.RegisterSystemModule(&setup1.Module{})
-		module.RegisterSystemModule(&ui.UIModule{})
+		module.RegisterSystemModule(uiModule)
 
 		var initFunc= func() {
 			module.RegisterSystemModule(&stats.SimpleStatsModule{})
@@ -85,13 +87,16 @@ func main() {
 			module.RegisterSystemModule(&pipeline.PipeModule{})
 			module.RegisterSystemModule(&task.TaskModule{})
 			module.RegisterSystemModule(&agent.AgentModule{})
-			module.RegisterUserPlugin(&metrics.MetricsModule{})
+			module.RegisterSystemModule(&metrics.MetricsModule{})
-			module.RegisterUserPlugin(&security.Module{})
+			module.RegisterSystemModule(&security.Module{})
 		}
 
 		if !global.Env().SetupRequired(){
 			initFunc()
 		}else{
+			for _, v := range modules {
+				v.Setup()
+			}
 			setup1.RegisterSetupCallback(initFunc)
 		}
 
@@ -123,9 +128,7 @@ func main() {
 
 		var initFunc= func() {
 			if global.Env().SetupRequired() {
-
 				for _, v := range modules {
-					v.Setup()
 					v.Start()
 				}
 			}

plugin/setup/setup.go

@@ -16,10 +16,14 @@ import (
 	"infini.sh/framework/core/util"
 	elastic2 "infini.sh/framework/modules/elastic"
 	elastic1 "infini.sh/framework/modules/elastic/common"
+	elastic3 "infini.sh/framework/modules/elastic/api"
 	"infini.sh/framework/modules/security"
+	"io"
 	"net/http"
+	uri2 "net/url"
 	"path"
 	"runtime"
+	"github.com/valyala/fasttemplate"
 	"time"
 )
 
@@ -43,6 +47,7 @@ func (module *Module) Setup() {
 
 	api.HandleAPIMethod(api.POST, "/setup/_validate", module.validate)
 	api.HandleAPIMethod(api.POST, "/setup/_initialize", module.initialize)
+	elastic3.InitTestAPI()
 }
 
 var setupFinishedCallback= []func() {}
@@ -65,6 +70,8 @@ func (module *Module) Stop() error {
 
 type SetupRequest struct {
 	Cluster struct {
+		Host string `json:"host"`
+		Schema string `json:"schema"`
 		Endpoint string `json:"endpoint"`
 		Username string `json:"username"`
 		Password string `json:"password"`
@@ -96,6 +103,9 @@ func (module *Module) validate(w http.ResponseWriter, r *http.Request, ps httpro
 	var code int
 	code=200
 	defer func() {
+
+		global.Env().CheckSetup()
+
 		result := util.MapStr{}
 		result["success"]=success
 
@@ -171,12 +181,17 @@ func (module *Module) validate(w http.ResponseWriter, r *http.Request, ps httpro
 
 	if indices != nil && len(*indices) > 0 {
 		buff := bytes.Buffer{}
+		tipBuff := bytes.Buffer{}
 		for k, _ := range *indices {
 			buff.WriteString(k)
 			buff.WriteString("\n")
+
+			tipBuff.WriteString("DELETE ")
+			tipBuff.WriteString(k)
+			tipBuff.WriteString("\n")
 		}
 		errType = IndicesExists
-		fixTips="DELETE "+util.TrimSpaces(cfg1.IndexPrefix) + "*"
+		fixTips=tipBuff.String()
 		panic(errors.Errorf("there are following indices exists in target elasticsearch: \n%v", buff.String()))
 	}
 
@@ -200,10 +215,16 @@ func (module *Module) initTempClient(r *http.Request) (error, elastic.API,SetupR
 		return err,nil,request
 	}
 
-	if request.Cluster.Endpoint==""{
+	if request.Cluster.Endpoint==""&&request.Cluster.Host==""{
 		panic("invalid configuration")
 	}
 
+	if request.Cluster.Endpoint==""{
+		if request.Cluster.Host!=""&&request.Cluster.Schema!=""{
+			request.Cluster.Endpoint=fmt.Sprintf("%v://%v",request.Cluster.Schema,request.Cluster.Host)
+		}
+	}
+
 	cfg = elastic.ElasticsearchConfig{
 		Enabled:  true,
 		Reserved: true,
@@ -214,6 +235,15 @@ func (module *Module) initTempClient(r *http.Request) (error, elastic.API,SetupR
 		},
 	}
 
+	if cfg.Endpoint!=""&&cfg.Host==""{
+		uri,err:=uri2.Parse(cfg.Endpoint)
+		if err!=nil{
+			panic(err)
+		}
+		cfg.Host=uri.Host
+		cfg.Schema=uri.Scheme
+	}
+
 	cfg.ID = tempID
 	cfg.Name = "INFINI_SYSTEM ("+util.PickRandomName()+")"
 	elastic.InitMetadata(&cfg, true)
@@ -243,6 +273,9 @@ func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps http
 	var code int
 	code=200
 	defer func() {
+
+		global.Env().CheckSetup()
+
 		result := util.MapStr{}
 		result["success"]=success
 
@@ -297,7 +330,7 @@ func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps http
 
 	//处理ORM
 	handler := elastic2.ElasticORM{Client: client, Config:cfg1 }
-	orm.Register("elastic_setup", handler)
+	orm.Register("elastic_setup_"+util.GetUUID(), handler)
 
 	//处理模版
 	elastic2.InitTemplate(true)
@@ -306,6 +339,39 @@ func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps http
 	//TEMPLATE_NAME
 	//INDEX_PREFIX
 
+	dslTplFile:=path.Join(global.Env().GetConfigDir(),"initialization.tpl")
+	dslFile:=path.Join(global.Env().GetConfigDir(),"initialization.dsl")
+
+	var dsl []byte
+	dsl,err=util.FileGetContent(dslTplFile)
+	if err!=nil{
+		panic(err)
+	}
+
+	if len(dsl)>0{
+		var tpl *fasttemplate.Template
+		tpl,err=fasttemplate.NewTemplate(string(dsl), "$[[", "]]")
+		if err!=nil{
+			panic(err)
+		}
+		if tpl!=nil{
+			output:=tpl.ExecuteFuncString(func(w io.Writer, tag string) (int, error) {
+				switch tag {
+				case "TEMPLATE_NAME":
+					return w.Write([]byte(cfg1.TemplateName))
+				case "INDEX_PREFIX":
+					return w.Write([]byte(cfg1.IndexPrefix))
+				}
+				panic(errors.Errorf("unknown tag: %v",tag))
+			})
+			_,err=util.FilePutContent(dslFile,output)
+			if err!=nil{
+				panic(err)
+			}
+		}
+	}
+
+
 	//处理索引
 	elastic2.InitSchema()
 	//init security