diff --git a/config/config.go b/config/config.go index 94d1e6d8..a3cfb9a2 100644 --- a/config/config.go +++ b/config/config.go @@ -3,7 +3,6 @@ package config import "infini.sh/framework/core/config" type AppConfig struct { - Elasticsearch string `config:"elasticsearch"` UI UIConfig `config:"ui"` Network config.NetworkConfig `config:"network"` TLSConfig config.TLSConfig `config:"tls"` diff --git a/config/initialization.tpl b/config/initialization.tpl index 0f5b22bd..6e284740 100644 --- a/config/initialization.tpl +++ b/config/initialization.tpl @@ -1,8 +1,8 @@ -PUT _template/.infini +PUT _template/$[[TEMPLATE_NAME]] { "order": 0, "index_patterns": [ - ".infini_*" + "$[[INDEX_PREFIX]]*" ], "settings": { "index": { @@ -41,7 +41,7 @@ PUT _template/.infini "aliases": {} } -PUT _ilm/policy/infini_metrics-30days-retention +PUT _ilm/policy/ilm_$[[INDEX_PREFIX]]metrics-30days-retention { "policy": { "phases": { @@ -69,18 +69,18 @@ PUT _ilm/policy/infini_metrics-30days-retention } } -PUT _template/.infini_metrics-rollover +PUT _template/$[[INDEX_PREFIX]]metrics-rollover { "order" : 100000, "index_patterns" : [ - ".infini_metrics*" + "$[[INDEX_PREFIX]]metrics*" ], "settings" : { "index" : { "format" : "7", "lifecycle" : { - "name" : "infini_metrics-30days-retention", - "rollover_alias" : ".infini_metrics" + "name" : "ilm_$[[INDEX_PREFIX]]metrics-30days-retention", + "rollover_alias" : "$[[INDEX_PREFIX]]metrics" }, "codec" : "best_compression", "number_of_shards" : "1", @@ -104,16 +104,14 @@ PUT _template/.infini_metrics-rollover } -# DELETE .infini_metrics -# DELETE .infini_metrics-00001 -PUT .infini_metrics-00001 +PUT $[[INDEX_PREFIX]]metrics-00001 { "settings": { - "index.lifecycle.rollover_alias":".infini_metrics" + "index.lifecycle.rollover_alias":"$[[INDEX_PREFIX]]metrics" , "refresh_interval": "5s" }, "aliases":{ - ".infini_metrics":{ + "$[[INDEX_PREFIX]]metrics":{ "is_write_index":true } } @@ -121,18 +119,18 @@ PUT .infini_metrics-00001 -PUT _template/.infini_alert-history-rollover +PUT _template/$[[INDEX_PREFIX]]alert-history-rollover { "order" : 100000, "index_patterns" : [ - ".infini_alert-history*" + "$[[INDEX_PREFIX]]alert-history*" ], "settings" : { "index" : { "format" : "7", "lifecycle" : { - "name" : "infini_metrics-30days-retention", - "rollover_alias" : ".infini_alert-history" + "name" : "ilm_$[[INDEX_PREFIX]]metrics-30days-retention", + "rollover_alias" : "$[[INDEX_PREFIX]]alert-history" }, "codec" : "best_compression", "number_of_shards" : "1", @@ -155,16 +153,15 @@ PUT _template/.infini_alert-history-rollover "aliases" : { } } -# DELETE .infini_alert-history -# DELETE .infini_alert-history-00001 -PUT .infini_alert-history-00001 + +PUT $[[INDEX_PREFIX]]alert-history-00001 { "settings": { - "index.lifecycle.rollover_alias":".infini_alert-history" + "index.lifecycle.rollover_alias":"$[[INDEX_PREFIX]]alert-history" , "refresh_interval": "5s" }, "aliases":{ - ".infini_alert-history":{ + "$[[INDEX_PREFIX]]alert-history":{ "is_write_index":true } }, @@ -278,20 +275,18 @@ PUT .infini_alert-history-00001 } - - -PUT _template/.infini_activities-rollover +PUT _template/$[[INDEX_PREFIX]]activities-rollover { "order" : 100000, "index_patterns" : [ - ".infini_activities*" + "$[[INDEX_PREFIX]]activities*" ], "settings" : { "index" : { "format" : "7", "lifecycle" : { - "name" : "infini_metrics-30days-retention", - "rollover_alias" : ".infini_activities" + "name" : "ilm_$[[INDEX_PREFIX]]metrics-30days-retention", + "rollover_alias" : "$[[INDEX_PREFIX]]activities" }, "codec" : "best_compression", "number_of_shards" : "1", @@ -314,17 +309,15 @@ PUT _template/.infini_activities-rollover "aliases" : { } } -#DELETE .infini_activities -#DELETE .infini_activities-00001 -PUT .infini_activities-00001 +PUT $[[INDEX_PREFIX]]activities-00001 { "settings": { - "index.lifecycle.rollover_alias":".infini_activities" + "index.lifecycle.rollover_alias":"$[[INDEX_PREFIX]]activities" , "refresh_interval": "5s" }, "aliases":{ - ".infini_activities":{ + "$[[INDEX_PREFIX]]activities":{ "is_write_index":true } } diff --git a/config/system_config.tpl b/config/system_config.tpl index c1f9aaa4..2e1d7811 100644 --- a/config/system_config.tpl +++ b/config/system_config.tpl @@ -3,7 +3,7 @@ elasticsearch: - id: $[[CLUSTER_ID]] name: $[[CLUSTER_ID]] enabled: true - monitored: false + monitored: true reserved: true endpoint: $[[CLUSTER_ENDPINT]] basic_auth: diff --git a/config/system_config.yml b/config/system_config.yml deleted file mode 100644 index 87531125..00000000 --- a/config/system_config.yml +++ /dev/null @@ -1,9 +0,0 @@ -configs.template: - - name: "system" - path: ./config/system_config.tpl - variable: - CLUSTER_ID: _setup_clustercd9s5brq50k75kji57tg - CLUSTER_ENDPINT: "http://192.168.3.188:9206" - CLUSTER_USER: "admin" - CLUSTER_PASS: "pass" - INDEX_PREFIX: ".infini_" \ No newline at end of file diff --git a/main.go b/main.go index 4a9c3119..f9b6a73e 100644 --- a/main.go +++ b/main.go @@ -69,13 +69,15 @@ func main() { modules=append(modules,&task.TaskModule{}) modules=append(modules,&agent.AgentModule{}) modules=append(modules,&metrics.MetricsModule{}) + modules=append(modules,&security.Module{}) + uiModule:=&ui.UIModule{} if app.Setup(func() { //load core modules first module.RegisterSystemModule(&setup1.Module{}) - module.RegisterSystemModule(&ui.UIModule{}) + module.RegisterSystemModule(uiModule) var initFunc= func() { module.RegisterSystemModule(&stats.SimpleStatsModule{}) @@ -85,13 +87,16 @@ func main() { module.RegisterSystemModule(&pipeline.PipeModule{}) module.RegisterSystemModule(&task.TaskModule{}) module.RegisterSystemModule(&agent.AgentModule{}) - module.RegisterUserPlugin(&metrics.MetricsModule{}) - module.RegisterUserPlugin(&security.Module{}) + module.RegisterSystemModule(&metrics.MetricsModule{}) + module.RegisterSystemModule(&security.Module{}) } if !global.Env().SetupRequired(){ initFunc() }else{ + for _, v := range modules { + v.Setup() + } setup1.RegisterSetupCallback(initFunc) } @@ -123,9 +128,7 @@ func main() { var initFunc= func() { if global.Env().SetupRequired() { - for _, v := range modules { - v.Setup() v.Start() } } diff --git a/plugin/setup/setup.go b/plugin/setup/setup.go index 7aa873e9..961f2926 100644 --- a/plugin/setup/setup.go +++ b/plugin/setup/setup.go @@ -16,10 +16,14 @@ import ( "infini.sh/framework/core/util" elastic2 "infini.sh/framework/modules/elastic" elastic1 "infini.sh/framework/modules/elastic/common" + elastic3 "infini.sh/framework/modules/elastic/api" "infini.sh/framework/modules/security" + "io" "net/http" + uri2 "net/url" "path" "runtime" + "github.com/valyala/fasttemplate" "time" ) @@ -43,6 +47,7 @@ func (module *Module) Setup() { api.HandleAPIMethod(api.POST, "/setup/_validate", module.validate) api.HandleAPIMethod(api.POST, "/setup/_initialize", module.initialize) + elastic3.InitTestAPI() } var setupFinishedCallback= []func() {} @@ -65,6 +70,8 @@ func (module *Module) Stop() error { type SetupRequest struct { Cluster struct { + Host string `json:"host"` + Schema string `json:"schema"` Endpoint string `json:"endpoint"` Username string `json:"username"` Password string `json:"password"` @@ -96,6 +103,9 @@ func (module *Module) validate(w http.ResponseWriter, r *http.Request, ps httpro var code int code=200 defer func() { + + global.Env().CheckSetup() + result := util.MapStr{} result["success"]=success @@ -171,12 +181,17 @@ func (module *Module) validate(w http.ResponseWriter, r *http.Request, ps httpro if indices != nil && len(*indices) > 0 { buff := bytes.Buffer{} + tipBuff := bytes.Buffer{} for k, _ := range *indices { buff.WriteString(k) buff.WriteString("\n") + + tipBuff.WriteString("DELETE ") + tipBuff.WriteString(k) + tipBuff.WriteString("\n") } errType = IndicesExists - fixTips="DELETE "+util.TrimSpaces(cfg1.IndexPrefix) + "*" + fixTips=tipBuff.String() panic(errors.Errorf("there are following indices exists in target elasticsearch: \n%v", buff.String())) } @@ -200,10 +215,16 @@ func (module *Module) initTempClient(r *http.Request) (error, elastic.API,SetupR return err,nil,request } - if request.Cluster.Endpoint==""{ + if request.Cluster.Endpoint==""&&request.Cluster.Host==""{ panic("invalid configuration") } + if request.Cluster.Endpoint==""{ + if request.Cluster.Host!=""&&request.Cluster.Schema!=""{ + request.Cluster.Endpoint=fmt.Sprintf("%v://%v",request.Cluster.Schema,request.Cluster.Host) + } + } + cfg = elastic.ElasticsearchConfig{ Enabled: true, Reserved: true, @@ -214,6 +235,15 @@ func (module *Module) initTempClient(r *http.Request) (error, elastic.API,SetupR }, } + if cfg.Endpoint!=""&&cfg.Host==""{ + uri,err:=uri2.Parse(cfg.Endpoint) + if err!=nil{ + panic(err) + } + cfg.Host=uri.Host + cfg.Schema=uri.Scheme + } + cfg.ID = tempID cfg.Name = "INFINI_SYSTEM ("+util.PickRandomName()+")" elastic.InitMetadata(&cfg, true) @@ -243,6 +273,9 @@ func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps http var code int code=200 defer func() { + + global.Env().CheckSetup() + result := util.MapStr{} result["success"]=success @@ -297,7 +330,7 @@ func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps http //处理ORM handler := elastic2.ElasticORM{Client: client, Config:cfg1 } - orm.Register("elastic_setup", handler) + orm.Register("elastic_setup_"+util.GetUUID(), handler) //处理模版 elastic2.InitTemplate(true) @@ -306,6 +339,39 @@ func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps http //TEMPLATE_NAME //INDEX_PREFIX + dslTplFile:=path.Join(global.Env().GetConfigDir(),"initialization.tpl") + dslFile:=path.Join(global.Env().GetConfigDir(),"initialization.dsl") + + var dsl []byte + dsl,err=util.FileGetContent(dslTplFile) + if err!=nil{ + panic(err) + } + + if len(dsl)>0{ + var tpl *fasttemplate.Template + tpl,err=fasttemplate.NewTemplate(string(dsl), "$[[", "]]") + if err!=nil{ + panic(err) + } + if tpl!=nil{ + output:=tpl.ExecuteFuncString(func(w io.Writer, tag string) (int, error) { + switch tag { + case "TEMPLATE_NAME": + return w.Write([]byte(cfg1.TemplateName)) + case "INDEX_PREFIX": + return w.Write([]byte(cfg1.IndexPrefix)) + } + panic(errors.Errorf("unknown tag: %v",tag)) + }) + _,err=util.FilePutContent(dslFile,output) + if err!=nil{ + panic(err) + } + } + } + + //处理索引 elastic2.InitSchema() //init security