From 9a91ca7b70f25daa67405421a3dbbc4b223b88f4 Mon Sep 17 00:00:00 2001 From: medcl Date: Mon, 24 Oct 2022 19:52:09 +0800 Subject: [PATCH 1/5] add skip to initialize api --- plugin/setup/setup.go | 152 +++++++++++++++++++++--------------------- 1 file changed, 77 insertions(+), 75 deletions(-) diff --git a/plugin/setup/setup.go b/plugin/setup/setup.go index b4d7984c..b21145ec 100644 --- a/plugin/setup/setup.go +++ b/plugin/setup/setup.go @@ -80,6 +80,7 @@ type SetupRequest struct { Password string `json:"password"` } `json:"cluster"` + Skip bool `json:"skip"` BootstrapUsername string `json:"bootstrap_username"` BootstrapPassword string `json:"bootstrap_password"` } @@ -336,93 +337,102 @@ func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps http handler := elastic2.ElasticORM{Client: client, Config:cfg1 } orm.Register("elastic_setup_"+util.GetUUID(), handler) - //处理模版 - elastic2.InitTemplate(true) + if !request.Skip{ + //处理模版 + elastic2.InitTemplate(true) - //处理生命周期 - //TEMPLATE_NAME - //INDEX_PREFIX - dslTplFile:=path.Join(global.Env().GetConfigDir(),"initialization.tpl") - dslFile:=path.Join(global.Env().GetConfigDir(),"initialization.dsl") + //处理生命周期 + //TEMPLATE_NAME + //INDEX_PREFIX + dslTplFile:=path.Join(global.Env().GetConfigDir(),"initialization.tpl") + dslFile:=path.Join(global.Env().GetConfigDir(),"initialization.dsl") - var dsl []byte - dsl,err=util.FileGetContent(dslTplFile) - if err!=nil{ - panic(err) - } - - var dslWriteSuccess=false - if len(dsl)>0{ - var tpl *fasttemplate.Template - tpl,err=fasttemplate.NewTemplate(string(dsl), "$[[", "]]") + var dsl []byte + dsl,err=util.FileGetContent(dslTplFile) if err!=nil{ panic(err) } - if tpl!=nil{ - output:=tpl.ExecuteFuncString(func(w io.Writer, tag string) (int, error) { - switch tag { - case "TEMPLATE_NAME": - return w.Write([]byte(cfg1.TemplateName)) - case "INDEX_PREFIX": - return w.Write([]byte(cfg1.IndexPrefix)) - case "RESOURCE_ID": - return w.Write([]byte(cfg.ID)) - case "RESOURCE_NAME": - return w.Write([]byte(cfg.Name)) - } - panic(errors.Errorf("unknown tag: %v",tag)) - }) - _,err=util.FilePutContent(dslFile,output) + + var dslWriteSuccess=false + if len(dsl)>0{ + var tpl *fasttemplate.Template + tpl,err=fasttemplate.NewTemplate(string(dsl), "$[[", "]]") if err!=nil{ panic(err) } - dslWriteSuccess=true + if tpl!=nil{ + output:=tpl.ExecuteFuncString(func(w io.Writer, tag string) (int, error) { + switch tag { + case "TEMPLATE_NAME": + return w.Write([]byte(cfg1.TemplateName)) + case "INDEX_PREFIX": + return w.Write([]byte(cfg1.IndexPrefix)) + case "RESOURCE_ID": + return w.Write([]byte(cfg.ID)) + case "RESOURCE_NAME": + return w.Write([]byte(cfg.Name)) + } + panic(errors.Errorf("unknown tag: %v",tag)) + }) + _,err=util.FilePutContent(dslFile,output) + if err!=nil{ + panic(err) + } + dslWriteSuccess=true + } } - } - if dslWriteSuccess{ - lines := util.FileGetLines(dslFile) - _,err,_:=replay.ReplayLines(pipeline.AcquireContext(),lines,cfg.Schema,cfg.Host) - if err!=nil{ - log.Error(err) + if dslWriteSuccess{ + lines := util.FileGetLines(dslFile) + _,err,_:=replay.ReplayLines(pipeline.AcquireContext(),lines,cfg.Schema,cfg.Host) + if err!=nil{ + log.Error(err) + } } - } + //处理索引 + elastic2.InitSchema() + //init security + security.InitSecurity() - //处理索引 - elastic2.InitSchema() - //init security - security.InitSecurity() - - //保存默认集群 - err=orm.Save(&cfg) - if err!=nil{ - panic(err) - } - - if request.BootstrapUsername!=""&&request.BootstrapPassword!=""{ - //Save bootstrap user - user:=rbac.User{} - user.ID="default_user_"+request.BootstrapUsername - user.Name=request.BootstrapUsername - user.NickName=request.BootstrapUsername - var hash []byte - hash, err = bcrypt.GenerateFromPassword([]byte(request.BootstrapPassword), bcrypt.DefaultCost) + //保存默认集群 + err=orm.Save(&cfg) if err!=nil{ panic(err) } - user.Password=string(hash) - role:=[]rbac.UserRole{} - role=append(role,rbac.UserRole{ - ID: rbac.RoleAdminName, - Name: rbac.RoleAdminName, - }) - user.Roles=role - err=orm.Save(&user) + if request.BootstrapUsername!=""&&request.BootstrapPassword!=""{ + //Save bootstrap user + user:=rbac.User{} + user.ID="default_user_"+request.BootstrapUsername + user.Name=request.BootstrapUsername + user.NickName=request.BootstrapUsername + var hash []byte + hash, err = bcrypt.GenerateFromPassword([]byte(request.BootstrapPassword), bcrypt.DefaultCost) + if err!=nil{ + panic(err) + } + + user.Password=string(hash) + role:=[]rbac.UserRole{} + role=append(role,rbac.UserRole{ + ID: rbac.RoleAdminName, + Name: rbac.RoleAdminName, + }) + user.Roles=role + err=orm.Save(&user) + if err!=nil{ + panic(err) + } + } + + + //disable builtin auth + err=api.DisableBuiltinUserAdmin() if err!=nil{ panic(err) } + } @@ -436,17 +446,9 @@ func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps http panic(err) } - //处理 ILM - //callback InvokeSetupCallback() - //disable builtin auth - err=api.DisableBuiltinUserAdmin() - if err!=nil{ - panic(err) - } - //place setup lock file setupLock:=path.Join(global.Env().GetDataDir(),".setup_lock") _,err=util.FilePutContent(setupLock,time.Now().String()) From 984111768218435bd4ab0e4257a11b5d6331e598 Mon Sep 17 00:00:00 2001 From: medcl Date: Mon, 24 Oct 2022 20:08:37 +0800 Subject: [PATCH 2/5] improve module init after setup --- main.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/main.go b/main.go index f9b6a73e..d70418fe 100644 --- a/main.go +++ b/main.go @@ -79,7 +79,7 @@ func main() { module.RegisterSystemModule(&setup1.Module{}) module.RegisterSystemModule(uiModule) - var initFunc= func() { + if !global.Env().SetupRequired(){ module.RegisterSystemModule(&stats.SimpleStatsModule{}) module.RegisterSystemModule(&elastic2.ElasticModule{}) module.RegisterSystemModule(&queue2.DiskQueue{}) @@ -89,15 +89,10 @@ func main() { module.RegisterSystemModule(&agent.AgentModule{}) module.RegisterSystemModule(&metrics.MetricsModule{}) module.RegisterSystemModule(&security.Module{}) - } - - if !global.Env().SetupRequired(){ - initFunc() }else{ for _, v := range modules { v.Setup() } - setup1.RegisterSetupCallback(initFunc) } api.RegisterAPI("") @@ -127,11 +122,7 @@ func main() { module.Start() var initFunc= func() { - if global.Env().SetupRequired() { - for _, v := range modules { - v.Start() - } - } + elastic2.InitTemplate(false) @@ -148,6 +139,12 @@ func main() { orm.RegisterSchemaWithIndexName(insight.Dashboard{}, "dashboard") api.RegisterSchema() + if global.Env().SetupRequired() { + for _, v := range modules { + v.Start() + } + } + task1.RunWithinGroup("initialize_alerting",func(ctx context.Context) error { err := alerting2.InitTasks() if err != nil { From be4e3f5b0baba37da14cfbbfc1d0dfb7d891e6e2 Mon Sep 17 00:00:00 2001 From: medcl Date: Tue, 25 Oct 2022 12:01:38 +0800 Subject: [PATCH 3/5] enable config auto reload by default --- console.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/console.yml b/console.yml index 23116695..62070ed8 100644 --- a/console.yml +++ b/console.yml @@ -1,4 +1,5 @@ path.configs: "config" +configs.auto_reload: true web: enabled: true From 523004b2d27ee88f974adcdd253551291d001b8b Mon Sep 17 00:00:00 2001 From: medcl Date: Tue, 25 Oct 2022 14:42:06 +0800 Subject: [PATCH 4/5] fix nil version during setup --- config/system_config.tpl | 1 + plugin/api/insight/metric_util.go | 5 +++++ plugin/setup/setup.go | 12 +++++++----- service/alerting/elasticsearch/engine.go | 5 +++++ 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/config/system_config.tpl b/config/system_config.tpl index 2e1d7811..0f8b93f8 100644 --- a/config/system_config.tpl +++ b/config/system_config.tpl @@ -2,6 +2,7 @@ elasticsearch: - id: $[[CLUSTER_ID]] name: $[[CLUSTER_ID]] + version: $[[CLUSTER_VER]] enabled: true monitored: true reserved: true diff --git a/plugin/api/insight/metric_util.go b/plugin/api/insight/metric_util.go index aa26e5d9..cf2b63d7 100644 --- a/plugin/api/insight/metric_util.go +++ b/plugin/api/insight/metric_util.go @@ -76,6 +76,11 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) { } } targetESVersion := elastic.GetMetadata(metric.ClusterId).Config.Version + + if targetESVersion==""{ + panic("invalid version") + } + intervalField, err := elastic.GetDateHistogramIntervalField(targetESVersion, metric.BucketSize) if err != nil { return nil, fmt.Errorf("get interval field error: %w", err) diff --git a/plugin/setup/setup.go b/plugin/setup/setup.go index b21145ec..a4885128 100644 --- a/plugin/setup/setup.go +++ b/plugin/setup/setup.go @@ -250,15 +250,17 @@ func (module *Module) initTempClient(r *http.Request) (error, elastic.API,SetupR cfg.ID = tempID cfg.Name = "INFINI_SYSTEM ("+util.PickRandomName()+")" - elastic.InitMetadata(&cfg, true) + client, err := elastic1.InitClientWithConfig(cfg) if err != nil { return err,nil,request } - + cfg.Version=client.GetVersion() + meta:=elastic.InitMetadata(&cfg, true) + meta.Config.Version=cfg.Version + elastic.SetMetadata(tempID,meta) elastic.UpdateConfig(cfg) elastic.UpdateClient(cfg, client) - cfg.Version=client.GetVersion() global.Register(elastic.GlobalSystemElasticsearchID,tempID) return err, client,request @@ -440,8 +442,8 @@ func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps http file:=path.Join(global.Env().GetConfigDir(),"system_config.yml") _,err=util.FilePutContent(file,fmt.Sprintf("configs.template:\n - name: \"system\"\n path: ./config/system_config.tpl\n variable:\n " + "CLUSTER_ID: %v\n CLUSTER_ENDPINT: \"%v\"\n " + - "CLUSTER_USER: \"%v\"\n CLUSTER_PASS: \"%v\"\n INDEX_PREFIX: \"%v\"", - tempID,cfg.Endpoint,cfg.BasicAuth.Username,cfg.BasicAuth.Password,cfg1.IndexPrefix )) + "CLUSTER_USER: \"%v\"\n CLUSTER_PASS: \"%v\"\n CLUSTER_VER: \"%v\"\n INDEX_PREFIX: \"%v\"", + tempID,cfg.Endpoint,cfg.BasicAuth.Username,cfg.BasicAuth.Password,cfg.Version,cfg1.IndexPrefix )) if err!=nil{ panic(err) } diff --git a/service/alerting/elasticsearch/engine.go b/service/alerting/elasticsearch/engine.go index 8aeb8363..81dfd1cf 100644 --- a/service/alerting/elasticsearch/engine.go +++ b/service/alerting/elasticsearch/engine.go @@ -62,6 +62,11 @@ func (engine *Engine) GenerateQuery(rule *alerting.Rule, filterParam *alerting.F if filterParam != nil && filterParam.BucketSize != "" { periodInterval = filterParam.BucketSize } + + if targetESVersion==""{ + panic("invalid version") + } + intervalField, err := elastic.GetDateHistogramIntervalField(targetESVersion, periodInterval ) if err != nil { return nil, fmt.Errorf("get interval field error: %w", err) From ca46c22df480f1200e8f3b818b8de06818041abd Mon Sep 17 00:00:00 2001 From: medcl Date: Tue, 25 Oct 2022 16:01:12 +0800 Subject: [PATCH 5/5] fix nil exception --- plugin/setup/setup.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/plugin/setup/setup.go b/plugin/setup/setup.go index a4885128..5671589e 100644 --- a/plugin/setup/setup.go +++ b/plugin/setup/setup.go @@ -250,17 +250,15 @@ func (module *Module) initTempClient(r *http.Request) (error, elastic.API,SetupR cfg.ID = tempID cfg.Name = "INFINI_SYSTEM ("+util.PickRandomName()+")" - + elastic.InitMetadata(&cfg, true) client, err := elastic1.InitClientWithConfig(cfg) if err != nil { return err,nil,request } - cfg.Version=client.GetVersion() - meta:=elastic.InitMetadata(&cfg, true) - meta.Config.Version=cfg.Version - elastic.SetMetadata(tempID,meta) + elastic.UpdateConfig(cfg) elastic.UpdateClient(cfg, client) + cfg.Version=client.GetVersion() global.Register(elastic.GlobalSystemElasticsearchID,tempID) return err, client,request