From 5d15f2be8bb7ace0aaf460f93204c5fdde2ae9ec Mon Sep 17 00:00:00 2001 From: medcl Date: Sat, 16 Sep 2023 17:42:38 +0800 Subject: [PATCH 01/36] cleanup app config --- config/config.go | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/config/config.go b/config/config.go index a3cfb9a2..43a3aabb 100644 --- a/config/config.go +++ b/config/config.go @@ -3,22 +3,12 @@ package config import "infini.sh/framework/core/config" type AppConfig struct { + config.APIConfig UI UIConfig `config:"ui"` - Network config.NetworkConfig `config:"network"` - TLSConfig config.TLSConfig `config:"tls"` } type UIConfig struct { - Enabled bool `config:"enabled"` LocalPath string `config:"path"` LocalEnabled bool `config:"local"` VFSEnabled bool `config:"vfs"` - APIEndpoint string `config:"api_endpoint"` -} - -func (config *AppConfig) GetSchema() string { - if config.TLSConfig.TLSEnabled { - return "https" - } - return "http" } From 1b0022a86d27a3ea3658745c0c4026f0d075fff1 Mon Sep 17 00:00:00 2001 From: medcl Date: Sat, 16 Sep 2023 17:42:51 +0800 Subject: [PATCH 02/36] fix get nil endpoint --- bootstrap_check.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap_check.go b/bootstrap_check.go index fca79edc..ac7792cc 100644 --- a/bootstrap_check.go +++ b/bootstrap_check.go @@ -52,7 +52,7 @@ func checkElasticsearchRequirements() error{ if targetEsConfig == nil { return fmt.Errorf("cluster config %s was not found", esID) } - var req = util.NewGetRequest(targetEsConfig.Endpoint, nil) + var req = util.NewGetRequest(targetEsConfig.GetAnyEndpoint(), nil) if targetEsConfig.BasicAuth != nil { req.SetBasicAuth(targetEsConfig.BasicAuth.Username, targetEsConfig.BasicAuth.Password) } From 36737914671d4b44ec123e1d2a23f65839ca556f Mon Sep 17 00:00:00 2001 From: medcl Date: Sat, 16 Sep 2023 17:45:33 +0800 Subject: [PATCH 03/36] temp commit --- config/ingest_config.tpl | 90 ++ config/install_agent.tpl | 22 +- config/system_config.tpl | 34 +- config/task_config.tpl | 84 ++ console.yml | 1 - main.go | 12 +- model/instance.go | 44 +- modules/agent/agent.go | 158 --- modules/agent/api/host.go | 453 ++++--- modules/agent/api/init.go | 55 +- modules/agent/api/instance.go | 1143 ----------------- modules/agent/api/log.go | 20 +- modules/agent/api/setup.go | 155 --- modules/agent/client/client.go | 79 +- modules/agent/common/helper.go | 138 +- modules/agent/model/config.go | 20 +- modules/agent/state/state.go | 49 +- plugin/api/gateway/api.go | 92 +- plugin/api/gateway/instance.go | 946 +++++++------- plugin/api/gateway/websocket_proxy.go | 186 --- plugin/setup/setup.go | 6 +- plugin/task_manager/common_api.go | 4 +- plugin/task_manager/model/scheduler.go | 5 +- .../pipeline_task/pipeline_task.go | 6 +- plugin/task_manager/scheduler/scheduler.go | 14 +- service/alerting/env.go | 4 +- 26 files changed, 1101 insertions(+), 2719 deletions(-) create mode 100644 config/ingest_config.tpl create mode 100644 config/task_config.tpl delete mode 100644 modules/agent/agent.go delete mode 100644 modules/agent/api/setup.go delete mode 100644 plugin/api/gateway/websocket_proxy.go diff --git a/config/ingest_config.tpl b/config/ingest_config.tpl new file mode 100644 index 00000000..d93dffa9 --- /dev/null +++ b/config/ingest_config.tpl @@ -0,0 +1,90 @@ +elasticsearch: + - name: $[[INGEST_CLUSTER_ID]] + enabled: true + endpoints: $[[INGEST_CLUSTER_ENDPOINT]] + discovery: + enabled: false + basic_auth: + username: $[[INGEST_CLUSTER_USERNAME]] + password: $[[keystore.ingest_cluster_password]] + +metrics: + enabled: true + queue: metrics + network: + enabled: true + summary: true + details: true + memory: + metrics: + - swap + - memory + disk: + metrics: + - iops + - usage + cpu: + metrics: + - idle + - system + - user + - iowait + - load + instance: + enabled: true + +elastic: + availability_check: + enabled: false + +pipeline: + - name: merge_logs + auto_start: true + keep_running: true + processor: + - indexing_merge: + index_name: ".infini_logs" + elasticsearch: "$[[INGEST_CLUSTER_ID]]" + input_queue: "logs" + idle_timeout_in_seconds: 10 + output_queue: + name: "merged_requests" + worker_size: 1 + bulk_size_in_mb: 5 + - name: merge_metrics + auto_start: true + keep_running: true + processor: + - indexing_merge: + elasticsearch: "$[[INGEST_CLUSTER_ID]]" + index_name: ".infini_metrics" + input_queue: "metrics" + output_queue: + name: "merged_requests" + worker_size: 1 + bulk_size_in_mb: 5 + - name: ingest_merged_requests + auto_start: true + keep_running: true + processor: + - bulk_indexing: + max_worker_size: 1 + bulk: + batch_size_in_mb: 5 + batch_size_in_docs: 5000 + max_retry_times: 0 + invalid_queue: "" + response_handle: + include_index_stats: false + include_action_stats: false + output_bulk_stats: false + include_error_details: false + save_error_results: false + save_success_results: false + save_busy_results: false + consumer: + fetch_max_messages: 5 + queues: + type: indexing_merge + when: + cluster_available: ["$[[INGEST_CLUSTER_ID]]"] \ No newline at end of file diff --git a/config/install_agent.tpl b/config/install_agent.tpl index d7174559..e764d685 100644 --- a/config/install_agent.tpl +++ b/config/install_agent.tpl @@ -220,6 +220,23 @@ path.data: data path.logs: log path.configs: config +resource_limit.cpu.max_num_of_cpus: 1 +resource_limit.memory.max_in_bytes: 533708800 + +stats: + include_storage_stats_in_api: false + +disk_queue: + max_msg_size: 20485760 + max_bytes_per_file: 20485760 + max_used_bytes: 524288000 + retention.max_num_of_local_files: 1 + compress: + idle_threshold: 0 + num_of_files_decompress_ahead: 0 + segment: + enabled: true + api: enabled: true tls: @@ -232,11 +249,12 @@ api: binding: \$[[env.API_BINDING]] badger: + value_threshold: 1024 + mem_table_size: 1048576 value_log_max_entries: 1000000 value_log_file_size: 104857600 - value_threshold: 1024 -agent: +node: major_ip_pattern: ".*" EOF } diff --git a/config/system_config.tpl b/config/system_config.tpl index 23c4c338..d78fdde6 100644 --- a/config/system_config.tpl +++ b/config/system_config.tpl @@ -7,7 +7,7 @@ elasticsearch: enabled: true monitored: true reserved: true - endpoint: $[[CLUSTER_ENDPINT]] + endpoint: $[[CLUSTER_ENDPOINT]] discovery: enabled: false basic_auth: @@ -17,7 +17,7 @@ elasticsearch: elastic.elasticsearch: $[[CLUSTER_ID]] pipeline: - - name: indexing_merge + - name: merge_metrics auto_start: true keep_running: true processor: @@ -31,22 +31,7 @@ pipeline: tag: "metrics" worker_size: 1 bulk_size_in_mb: 5 - - name: consume-metrics_requests - auto_start: true - keep_running: true - processor: - - bulk_indexing: - bulk: - compress: true - batch_size_in_mb: 5 - batch_size_in_docs: 5000 - consumer: - fetch_max_messages: 100 - queues: - type: indexing_merge - tag: "metrics" - when: - cluster_available: ["$[[CLUSTER_ID]]"] + - name: metadata_ingest auto_start: true keep_running: true @@ -91,7 +76,7 @@ pipeline: when: cluster_available: ["$[[CLUSTER_ID]]"] - - name: logging_indexing_merge + - name: merge_logging auto_start: true keep_running: true processor: @@ -106,19 +91,22 @@ pipeline: tag: "request_logging" worker_size: 1 bulk_size_in_kb: 1 - - name: consume-logging_requests + + - name: ingest_merged_requests auto_start: true keep_running: true + retry_delay_in_ms: 5000 + max_running_in_ms: 30000 processor: - bulk_indexing: + idle_timeout_in_seconds: 5 bulk: compress: true - batch_size_in_mb: 1 - batch_size_in_docs: 1 + batch_size_in_mb: 10 + batch_size_in_docs: 1000 consumer: fetch_max_messages: 100 queues: type: indexing_merge - tag: "request_logging" when: cluster_available: ["$[[CLUSTER_ID]]"] \ No newline at end of file diff --git a/config/task_config.tpl b/config/task_config.tpl new file mode 100644 index 00000000..c5abc715 --- /dev/null +++ b/config/task_config.tpl @@ -0,0 +1,84 @@ +elasticsearch: + - id: $[[CLUSTER_ID]] + name: $[[CLUSTER_ID]] + enabled: true + endpoint: $[[CLUSTER_ENDPOINT]] + discovery: + enabled: false + basic_auth: + username: $[[CLUSTER_USERNAME]] + password: $[[keystore.$[[CLUSTER_ID]]_password]] + +pipeline: +#clsuter level metrics +- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + keep_running: true + singleton: true + name: collect_$[[CLUSTER_ID]]_es_cluster_stats + retry_delay_in_ms: 10000 + processor: + - es_cluster_stats: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + when: + cluster_available: ["$[[CLUSTER_ID]]"] + +- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + keep_running: true + singleton: true + name: collect_$[[CLUSTER_ID]]_es_index_stats + retry_delay_in_ms: 10000 + processor: + - es_index_stats: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + when: + cluster_available: ["$[[CLUSTER_ID]]"] + +- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + keep_running: true + singleton: true + name: collect_$[[CLUSTER_ID]]_es_cluster_health + retry_delay_in_ms: 10000 + processor: + - es_cluster_health: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + when: + cluster_available: ["$[[CLUSTER_ID]]"] + +#node level metrics +- auto_start: $[[NODE_LEVEL_TASKS_ENABLED]] + enabled: $[[NODE_LEVEL_TASKS_ENABLED]] + keep_running: true + name: collect_$[[CLUSTER_ID]]_es_node_stats + retry_delay_in_ms: 10000 + processor: + - es_node_stats: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + when: + cluster_available: ["$[[CLUSTER_ID]]"] + +#node logs +- auto_start: $[[NODE_LEVEL_TASKS_ENABLED]] + enabled: $[[NODE_LEVEL_TASKS_ENABLED]] + keep_running: true + name: collect_$[[CLUSTER_ID]]_es_logs + retry_delay_in_ms: 10000 + processor: + - es_logs_processor: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + logs_path: $[[NODE_LOGS_PATH]] + queue_name: logs + when: + cluster_available: ["$[[CLUSTER_ID]]"] diff --git a/console.yml b/console.yml index 2faba170..b10556bd 100644 --- a/console.yml +++ b/console.yml @@ -52,7 +52,6 @@ elastic: metrics: enabled: true - major_ip_pattern: "192.*" queue: metrics elasticsearch: enabled: true diff --git a/main.go b/main.go index f7d71d81..dd01f93e 100644 --- a/main.go +++ b/main.go @@ -5,6 +5,7 @@ import ( "errors" _ "expvar" "infini.sh/console/plugin/api/email" + model2 "infini.sh/framework/core/model" _ "time/tzdata" log "github.com/cihub/seelog" @@ -12,7 +13,6 @@ import ( "infini.sh/console/model" "infini.sh/console/model/alerting" "infini.sh/console/model/insight" - "infini.sh/console/modules/agent" _ "infini.sh/console/plugin" setup1 "infini.sh/console/plugin/setup" alerting2 "infini.sh/console/service/alerting" @@ -37,6 +37,7 @@ import ( _ "infini.sh/framework/plugins" api2 "infini.sh/gateway/api" _ "infini.sh/gateway/proxy" + _ "infini.sh/framework/plugins/managed" ) var appConfig *config.AppConfig @@ -70,7 +71,6 @@ func main() { modules = append(modules, module.ModuleItem{Value: &task.TaskModule{}, Priority: 1}) modules = append(modules, module.ModuleItem{Value: &metrics.MetricsModule{}, Priority: 1}) modules = append(modules, module.ModuleItem{Value: &security.Module{}, Priority: 1}) - modules = append(modules, module.ModuleItem{Value: &agent.AgentModule{}, Priority: 100}) uiModule := &ui.UIModule{} @@ -122,11 +122,11 @@ func main() { elastic2.InitTemplate(false) - //orm.RegisterSchemaWithIndexName(model.Dict{}, "dict") + //orm.RegisterSchema(model.Dict{}, "dict") orm.RegisterSchemaWithIndexName(elastic.View{}, "view") orm.RegisterSchemaWithIndexName(elastic.CommonCommand{}, "commands") - //orm.RegisterSchemaWithIndexName(elastic.TraceTemplate{}, "trace-template") - orm.RegisterSchemaWithIndexName(model.Instance{}, "instance") + //orm.RegisterSchema(elastic.TraceTemplate{}, "trace-template") + //orm.RegisterSchema(model.Instance{}, "instance") orm.RegisterSchemaWithIndexName(alerting.Rule{}, "alert-rule") orm.RegisterSchemaWithIndexName(alerting.Alert{}, "alert-history") orm.RegisterSchemaWithIndexName(alerting.AlertMessage{}, "alert-message") @@ -138,6 +138,8 @@ func main() { orm.RegisterSchemaWithIndexName(model.Layout{}, "layout") orm.RegisterSchemaWithIndexName(model.Notification{}, "notification") orm.RegisterSchemaWithIndexName(model.EmailServer{}, "email-server") + orm.RegisterSchemaWithIndexName(model2.Instance{}, "instance") + api.RegisterSchema() if global.Env().SetupRequired() { diff --git a/model/instance.go b/model/instance.go index 81b28946..9b7921e2 100644 --- a/model/instance.go +++ b/model/instance.go @@ -11,26 +11,15 @@ import ( "net/http" "time" - "infini.sh/framework/core/agent" - "infini.sh/framework/core/orm" + "infini.sh/framework/core/model" "infini.sh/framework/core/util" "infini.sh/framework/modules/pipeline" ) - -type Instance struct { - orm.ORMObjectBase - - //InstanceID string `json:"instance_id,omitempty" elastic_mapping:"instance_id: { type: keyword }"` - Name string `json:"name,omitempty" elastic_mapping:"name:{type:keyword,fields:{text: {type: text}}}"` - Endpoint string `json:"endpoint,omitempty" elastic_mapping:"endpoint: { type: keyword }"` - Version map[string]interface{} `json:"version,omitempty" elastic_mapping:"version: { type: object }"` - BasicAuth agent.BasicAuth `config:"basic_auth" json:"basic_auth,omitempty" elastic_mapping:"basic_auth:{type:object}"` - Owner string `json:"owner,omitempty" config:"owner" elastic_mapping:"owner:{type:keyword}"` - Tags []string `json:"tags,omitempty"` - Description string `json:"description,omitempty" config:"description" elastic_mapping:"description:{type:keyword}"` +type TaskWorker struct { + model.Instance } -func (inst *Instance) CreatePipeline(body []byte) error { +func (inst *TaskWorker) CreatePipeline(body []byte) error { req := &util.Request{ Method: http.MethodPost, Body: body, @@ -39,7 +28,7 @@ func (inst *Instance) CreatePipeline(body []byte) error { return inst.doRequest(req, nil) } -func (inst *Instance) StopPipeline(ctx context.Context, pipelineID string) error { +func (inst *TaskWorker) StopPipeline(ctx context.Context, pipelineID string) error { req := &util.Request{ Method: http.MethodPost, Url: fmt.Sprintf("%s/pipeline/task/%s/_stop", inst.Endpoint, pipelineID), @@ -48,13 +37,13 @@ func (inst *Instance) StopPipeline(ctx context.Context, pipelineID string) error return inst.doRequest(req, nil) } -func (inst *Instance) StopPipelineWithTimeout(pipelineID string, duration time.Duration) error { +func (inst *TaskWorker) StopPipelineWithTimeout(pipelineID string, duration time.Duration) error { ctx, cancel := context.WithTimeout(context.Background(), duration) defer cancel() return inst.StopPipeline(ctx, pipelineID) } -func (inst *Instance) StartPipeline(pipelineID string) error { +func (inst *TaskWorker) StartPipeline(pipelineID string) error { req := &util.Request{ Method: http.MethodPost, Url: fmt.Sprintf("%s/pipeline/task/%s/_start", inst.Endpoint, pipelineID), @@ -62,7 +51,7 @@ func (inst *Instance) StartPipeline(pipelineID string) error { return inst.doRequest(req, nil) } -func (inst *Instance) DeletePipeline(pipelineID string) error { +func (inst *TaskWorker) DeletePipeline(pipelineID string) error { req := &util.Request{ Method: http.MethodDelete, Url: fmt.Sprintf("%s/pipeline/task/%s", inst.Endpoint, pipelineID), @@ -70,7 +59,7 @@ func (inst *Instance) DeletePipeline(pipelineID string) error { return inst.doRequest(req, nil) } -func (inst *Instance) GetPipeline(pipelineID string) (*pipeline.PipelineStatus, error) { +func (inst *TaskWorker) GetPipeline(pipelineID string) (*pipeline.PipelineStatus, error) { if pipelineID == "" { return nil, errors.New("invalid pipelineID") } @@ -89,7 +78,7 @@ func (inst *Instance) GetPipeline(pipelineID string) (*pipeline.PipelineStatus, return &res, nil } -func (inst *Instance) GetPipelinesByIDs(pipelineIDs []string) (pipeline.GetPipelinesResponse, error) { +func (inst *TaskWorker) GetPipelinesByIDs(pipelineIDs []string) (pipeline.GetPipelinesResponse, error) { body := util.MustToJSONBytes(util.MapStr{ "ids": pipelineIDs, }) @@ -106,7 +95,7 @@ func (inst *Instance) GetPipelinesByIDs(pipelineIDs []string) (pipeline.GetPipel return res, err } -func (inst *Instance) DeleteQueueBySelector(selector util.MapStr) error { +func (inst *TaskWorker) DeleteQueueBySelector(selector util.MapStr) error { req := &util.Request{ Method: http.MethodDelete, Url: fmt.Sprintf("%s/queue/_search", inst.Endpoint), @@ -117,7 +106,7 @@ func (inst *Instance) DeleteQueueBySelector(selector util.MapStr) error { return inst.doRequest(req, nil) } -func (inst *Instance) DeleteQueueConsumersBySelector(selector util.MapStr) error { +func (inst *TaskWorker) DeleteQueueConsumersBySelector(selector util.MapStr) error { req := &util.Request{ Method: http.MethodDelete, Url: fmt.Sprintf("%s/queue/consumer/_search", inst.Endpoint), @@ -128,21 +117,22 @@ func (inst *Instance) DeleteQueueConsumersBySelector(selector util.MapStr) error return inst.doRequest(req, nil) } -func (inst *Instance) TryConnect(ctx context.Context) error { +func (inst *TaskWorker) TryConnect(ctx context.Context) error { req := &util.Request{ Method: http.MethodGet, - Url: fmt.Sprintf("%s/_framework/api/_info", inst.Endpoint), + Url: fmt.Sprintf("%s/_info", inst.Endpoint), Context: ctx, } return inst.doRequest(req, nil) } -func (inst *Instance) TryConnectWithTimeout(duration time.Duration) error { + +func (inst *TaskWorker) TryConnectWithTimeout(duration time.Duration) error { ctx, cancel := context.WithTimeout(context.Background(), duration) defer cancel() return inst.TryConnect(ctx) } -func (inst *Instance) doRequest(req *util.Request, resBody interface{}) error { +func (inst *TaskWorker) doRequest(req *util.Request, resBody interface{}) error { req.SetBasicAuth(inst.BasicAuth.Username, inst.BasicAuth.Password) result, err := util.ExecuteRequest(req) if err != nil { diff --git a/modules/agent/agent.go b/modules/agent/agent.go deleted file mode 100644 index 9b86e2d9..00000000 --- a/modules/agent/agent.go +++ /dev/null @@ -1,158 +0,0 @@ -/* Copyright © INFINI Ltd. All rights reserved. - * Web: https://infinilabs.com - * Email: hello#infini.ltd */ - -package agent - -import ( - "fmt" - log "github.com/cihub/seelog" - "infini.sh/console/modules/agent/api" - "infini.sh/console/modules/agent/client" - "infini.sh/console/modules/agent/common" - "infini.sh/console/modules/agent/model" - "infini.sh/console/modules/agent/state" - "infini.sh/framework/core/agent" - "infini.sh/framework/core/credential" - "infini.sh/framework/core/elastic" - "infini.sh/framework/core/env" - "infini.sh/framework/core/host" - "infini.sh/framework/core/kv" - "infini.sh/framework/core/orm" - "infini.sh/framework/core/util" - "time" -) - -func (module *AgentModule) Name() string { - return "agent" -} - -func (module *AgentModule) Setup() { - module.AgentConfig.Enabled = true - module.AgentConfig.StateManager.Enabled = true - exists, err := env.ParseConfig("agent", &module.AgentConfig) - if exists && err != nil { - panic(err) - } - if module.AgentConfig.Enabled { - api.Init() - } -} -func (module *AgentModule) Start() error { - if !module.AgentConfig.Enabled { - return nil - } - orm.RegisterSchemaWithIndexName(agent.Instance{}, "agent") - orm.RegisterSchemaWithIndexName(agent.ESNodeInfo{}, "agent-node") - orm.RegisterSchemaWithIndexName(host.HostInfo{}, "host") - orm.RegisterSchemaWithIndexName(agent.Setting{}, "agent-setting") - var ( - executor client.Executor - err error - caFile string - caKey string - ) - if module.AgentConfig.Setup != nil { - caFile = module.AgentConfig.Setup.CACertFile - caKey = module.AgentConfig.Setup.CAKeyFile - } - if caFile == "" && caKey == "" { - caFile, caKey, err = common.GetOrInitDefaultCaCerts() - if err != nil { - panic(err) - } - } - executor, err = client.NewMTLSExecutor(caFile, caKey) - if err != nil { - panic(err) - } - agClient := &client.Client{ - Executor: executor, - } - client.RegisterClient(agClient) - - if module.AgentConfig.StateManager.Enabled { - onlineAgentIDs, err := common.GetLatestOnlineAgentIDs(nil, 60) - if err != nil { - log.Error(err) - } - agents, err := common.LoadAgentsFromES("") - if err != nil { - log.Error(err) - } - agentIds := map[string]string{} - for _, ag := range agents { - if _, ok := onlineAgentIDs[ag.ID]; ok { - agentIds[ag.ID] = "online" - } - } - credential.RegisterChangeEvent(func(cred *credential.Credential) { - var effectsClusterIDs []string - elastic.WalkConfigs(func(key, value interface{}) bool { - if cfg, ok := value.(*elastic.ElasticsearchConfig); ok { - if cfg.CredentialID == cred.ID { - effectsClusterIDs = append(effectsClusterIDs, cfg.ID) - } - } - return true - }) - if len(effectsClusterIDs) > 0 { - queryDsl := util.MapStr{ - "query": util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "terms": util.MapStr{ - "metadata.labels.cluster_id": effectsClusterIDs, - }, - }, - }, - }, - }, - "script": util.MapStr{ - "source": fmt.Sprintf("ctx._source['updated'] = '%s'", time.Now().Format(time.RFC3339Nano)), - }, - } - err = orm.UpdateBy(agent.Setting{}, util.MustToJSONBytes(queryDsl)) - if err != nil { - log.Error(err) - } - } - //check ingest cluster credential - if module.AgentConfig.Setup != nil && module.AgentConfig.Setup.IngestClusterCredentialID == cred.ID { - agents, err = common.LoadAgentsFromES("") - if err != nil { - log.Error(err) - return - } - for _, ag := range agents { - err = kv.AddValue(model.KVAgentIngestConfigChanged, []byte(ag.ID), []byte("1")) - if err != nil { - log.Error(err) - } - } - } - }) - - sm := state.NewStateManager(time.Second*30, "agent_state", agentIds, agClient) - state.RegisterStateManager(sm) - go sm.LoopState() - } - return nil -} - -func (module *AgentModule) Stop() error { - if !module.AgentConfig.Enabled { - return nil - } - log.Info("start to stop agent module") - if module.AgentConfig.StateManager.Enabled { - state.GetStateManager().Stop() - } - log.Info("agent module was stopped") - return nil -} - -type AgentModule struct { - model.AgentConfig -} diff --git a/modules/agent/api/host.go b/modules/agent/api/host.go index 4fb061c8..f6341ebc 100644 --- a/modules/agent/api/host.go +++ b/modules/agent/api/host.go @@ -4,236 +4,223 @@ package api -import ( - "context" - "fmt" - log "github.com/cihub/seelog" - "infini.sh/console/modules/agent/state" - httprouter "infini.sh/framework/core/api/router" - "infini.sh/framework/core/host" - "infini.sh/framework/core/orm" - "infini.sh/framework/core/util" - "net/http" - "strings" - "time" -) - -func (h *APIHandler) enrollHost(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - var reqBody []struct { - AgentID string `json:"agent_id"` - HostName string `json:"host_name"` - IP string `json:"ip"` - Source string `json:"source"` - OSName string `json:"os_name"` - OSArch string `json:"os_arch"` - NodeID string `json:"node_uuid"` - } - err := h.DecodeJSON(req, &reqBody) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - errors := util.MapStr{} - for i, hi := range reqBody { - var ( - hostInfo *host.HostInfo - ) - switch hi.Source { - case "agent": - hostInfo, err = enrollHostFromAgent(hi.AgentID) - if err != nil { - errors[hi.IP] = util.MapStr{ - "error": err.Error(), - } - log.Error(err) - continue - } - hostInfo.IP = hi.IP - hostInfo.AgentID = hi.AgentID - err = orm.Create(nil, hostInfo) - if err != nil { - errors[hi.IP] = util.MapStr{ - "error": err.Error(), - } - log.Error(err) - continue - } - case "es_node": - hostInfo = &host.HostInfo{ - IP: hi.IP, - OSInfo: host.OS{ - Platform: hi.OSName, - KernelArch: hi.OSArch, - }, - NodeID: hi.NodeID, - } - default: - errors[hi.IP] = util.MapStr{ - "error": fmt.Errorf("unkonow source type"), - } - continue - } - hostInfo.Timestamp = time.Now() - var ctx *orm.Context - if i == len(reqBody) - 1 { - ctx = &orm.Context{ - Refresh: "wait_for", - } - } - hostInfo.OSInfo.Platform = strings.ToLower(hostInfo.OSInfo.Platform) - err = orm.Create(ctx, hostInfo) - if err != nil { - errors[hi.IP] = util.MapStr{ - "error": err.Error(), - } - log.Error(err) - continue - } - } - resBody := util.MapStr{ - "success": true, - } - if len(errors) > 0 { - resBody["errors"] = errors - resBody["success"] = false - } - - h.WriteJSON(w, resBody, http.StatusOK) -} - -func (h *APIHandler) deleteHost(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - hostID := ps.MustGetParameter("host_id") - hostInfo, err := getHost(hostID) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - ctx := orm.Context{ - Refresh: "wait_for", - } - err = orm.Delete(&ctx, hostInfo) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - h.WriteDeletedOKJSON(w, hostID) -} - -func (h *APIHandler) GetHostAgentInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - hostID := ps.MustGetParameter("host_id") - hostInfo, err := getHost(hostID) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - if hostInfo.AgentID == "" { - h.WriteJSON(w, util.MapStr{}, http.StatusOK) - return - } - - sm := state.GetStateManager() - ag, err := sm.GetAgent(hostInfo.AgentID) - if err != nil { - log.Error(err) - h.WriteJSON(w, util.MapStr{}, http.StatusOK) - return - } - aversion, err := ag.GetVersion() - if err == nil { - ag.Version = aversion - orm.Save(nil, ag) - } - h.WriteJSON(w, util.MapStr{ - "host_id": hostID, - "agent_id": ag.ID, - "version": ag.Version, - "status": hostInfo.AgentStatus, - "endpoint": ag.GetEndpoint(), - }, http.StatusOK) -} - -func getHost(hostID string) (*host.HostInfo, error){ - hostInfo := &host.HostInfo{} - hostInfo.ID = hostID - exists, err := orm.Get(hostInfo) - if err != nil { - return nil, fmt.Errorf("get host info error: %w", err) - } - if !exists { - return nil, fmt.Errorf("host [%s] not found", hostID) - } - return hostInfo, nil -} - -func (h *APIHandler) GetHostElasticProcess(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - hostID := ps.MustGetParameter("host_id") - hostInfo := &host.HostInfo{} - hostInfo.ID = hostID - exists, err := orm.Get(hostInfo) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - if !exists { - h.WriteError(w, fmt.Sprintf("host [%s] not found", hostID), http.StatusNotFound) - return - } - if hostInfo.AgentID == "" { - h.WriteJSON(w, util.MapStr{}, http.StatusOK) - return - } - sm := state.GetStateManager() - ag, err := sm.GetAgent(hostInfo.AgentID) - if err != nil { - log.Error(err) - h.WriteJSON(w, util.MapStr{}, http.StatusOK) - return - } - ctx,cancel := context.WithTimeout(context.Background(), time.Second * 10) - defer cancel() - esNodesInfo, err := sm.GetAgentClient().GetElasticsearchNodes(ctx, ag.GetEndpoint()) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - var processes []util.MapStr - for _, node := range esNodesInfo { - processes = append(processes, util.MapStr{ - "pid": node.ProcessInfo.PID, - "pid_status": node.ProcessInfo.Status, - "cluster_name": node.ClusterName, - "cluster_uuid": node.ClusterUuid, - "cluster_id": node.ClusterID, - "node_id": node.NodeUUID, - "node_name": node.NodeName, - "uptime_in_ms": time.Now().UnixMilli() - node.ProcessInfo.CreateTime, - }) - } - h.WriteJSON(w, util.MapStr{ - "elastic_processes": processes, - }, http.StatusOK) -} - -func enrollHostFromAgent(agentID string) (*host.HostInfo, error){ - sm := state.GetStateManager() - ag, err := sm.GetAgent(agentID) - if err != nil { - return nil, err - } - if ag == nil { - return nil, fmt.Errorf("can not found agent [%s]", agentID) - } - agentClient := sm.GetAgentClient() - hostInfo, err := agentClient.GetHostInfo(nil, ag.GetEndpoint()) - if err != nil { - return nil, err - } - hostInfo.AgentStatus = ag.Status - return hostInfo, nil -} \ No newline at end of file +// +//func (h *APIHandler) enrollHost(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +// var reqBody []struct { +// AgentID string `json:"agent_id"` +// HostName string `json:"host_name"` +// IP string `json:"ip"` +// Source string `json:"source"` +// OSName string `json:"os_name"` +// OSArch string `json:"os_arch"` +// NodeID string `json:"node_uuid"` +// } +// err := h.DecodeJSON(req, &reqBody) +// if err != nil { +// log.Error(err) +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// return +// } +// errors := util.MapStr{} +// for i, hi := range reqBody { +// var ( +// hostInfo *host.HostInfo +// ) +// switch hi.Source { +// case "agent": +// hostInfo, err = enrollHostFromAgent(hi.AgentID) +// if err != nil { +// errors[hi.IP] = util.MapStr{ +// "error": err.Error(), +// } +// log.Error(err) +// continue +// } +// hostInfo.IP = hi.IP +// hostInfo.AgentID = hi.AgentID +// err = orm.Create(nil, hostInfo) +// if err != nil { +// errors[hi.IP] = util.MapStr{ +// "error": err.Error(), +// } +// log.Error(err) +// continue +// } +// case "es_node": +// hostInfo = &host.HostInfo{ +// IP: hi.IP, +// OSInfo: host.OS{ +// Platform: hi.OSName, +// KernelArch: hi.OSArch, +// }, +// NodeID: hi.NodeID, +// } +// default: +// errors[hi.IP] = util.MapStr{ +// "error": fmt.Errorf("unkonow source type"), +// } +// continue +// } +// hostInfo.Timestamp = time.Now() +// var ctx *orm.Context +// if i == len(reqBody) - 1 { +// ctx = &orm.Context{ +// Refresh: "wait_for", +// } +// } +// hostInfo.OSInfo.Platform = strings.ToLower(hostInfo.OSInfo.Platform) +// err = orm.Create(ctx, hostInfo) +// if err != nil { +// errors[hi.IP] = util.MapStr{ +// "error": err.Error(), +// } +// log.Error(err) +// continue +// } +// } +// resBody := util.MapStr{ +// "success": true, +// } +// if len(errors) > 0 { +// resBody["errors"] = errors +// resBody["success"] = false +// } +// +// h.WriteJSON(w, resBody, http.StatusOK) +//} +// +//func (h *APIHandler) deleteHost(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +// hostID := ps.MustGetParameter("host_id") +// hostInfo, err := getHost(hostID) +// if err != nil { +// log.Error(err) +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// return +// } +// ctx := orm.Context{ +// Refresh: "wait_for", +// } +// err = orm.Delete(&ctx, hostInfo) +// if err != nil { +// log.Error(err) +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// return +// } +// h.WriteDeletedOKJSON(w, hostID) +//} +// +//func (h *APIHandler) GetHostAgentInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +// hostID := ps.MustGetParameter("host_id") +// hostInfo, err := getHost(hostID) +// if err != nil { +// log.Error(err) +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// return +// } +// if hostInfo.AgentID == "" { +// h.WriteJSON(w, util.MapStr{}, http.StatusOK) +// return +// } +// +// sm := state.GetStateManager() +// ag, err := sm.GetAgent(hostInfo.AgentID) +// if err != nil { +// log.Error(err) +// h.WriteJSON(w, util.MapStr{}, http.StatusOK) +// return +// } +// aversion, err := ag.GetVersion() +// if err == nil { +// ag.Version = aversion +// orm.Save(nil, ag) +// } +// h.WriteJSON(w, util.MapStr{ +// "host_id": hostID, +// "agent_id": ag.ID, +// "version": ag.Version, +// "status": hostInfo.AgentStatus, +// "endpoint": ag.GetEndpoint(), +// }, http.StatusOK) +//} +// +//func getHost(hostID string) (*host.HostInfo, error){ +// hostInfo := &host.HostInfo{} +// hostInfo.ID = hostID +// exists, err := orm.Get(hostInfo) +// if err != nil { +// return nil, fmt.Errorf("get host info error: %w", err) +// } +// if !exists { +// return nil, fmt.Errorf("host [%s] not found", hostID) +// } +// return hostInfo, nil +//} +// +//func (h *APIHandler) GetHostElasticProcess(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +// hostID := ps.MustGetParameter("host_id") +// hostInfo := &host.HostInfo{} +// hostInfo.ID = hostID +// exists, err := orm.Get(hostInfo) +// if err != nil { +// log.Error(err) +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// return +// } +// if !exists { +// h.WriteError(w, fmt.Sprintf("host [%s] not found", hostID), http.StatusNotFound) +// return +// } +// if hostInfo.AgentID == "" { +// h.WriteJSON(w, util.MapStr{}, http.StatusOK) +// return +// } +// sm := state.GetStateManager() +// ag, err := sm.GetAgent(hostInfo.AgentID) +// if err != nil { +// log.Error(err) +// h.WriteJSON(w, util.MapStr{}, http.StatusOK) +// return +// } +// ctx,cancel := context.WithTimeout(context.Background(), time.Second * 10) +// defer cancel() +// esNodesInfo, err := sm.GetAgentClient().GetElasticsearchNodes(ctx, ag.GetEndpoint()) +// if err != nil { +// log.Error(err) +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// return +// } +// var processes []util.MapStr +// for _, node := range esNodesInfo { +// processes = append(processes, util.MapStr{ +// "pid": node.ProcessInfo.PID, +// "pid_status": node.ProcessInfo.Status, +// "cluster_name": node.ClusterName, +// "cluster_uuid": node.ClusterUuid, +// "cluster_id": node.ClusterID, +// "node_id": node.NodeUUID, +// "node_name": node.NodeName, +// "uptime_in_ms": time.Now().UnixMilli() - node.ProcessInfo.CreateTime, +// }) +// } +// h.WriteJSON(w, util.MapStr{ +// "elastic_processes": processes, +// }, http.StatusOK) +//} +// +//func enrollHostFromAgent(agentID string) (*host.HostInfo, error){ +// sm := state.GetStateManager() +// ag, err := sm.GetAgent(agentID) +// if err != nil { +// return nil, err +// } +// if ag == nil { +// return nil, fmt.Errorf("can not found agent [%s]", agentID) +// } +// agentClient := sm.GetAgentClient() +// hostInfo, err := agentClient.GetHostInfo(nil, ag.GetEndpoint()) +// if err != nil { +// return nil, err +// } +// hostInfo.AgentStatus = ag.Status +// return hostInfo, nil +//} \ No newline at end of file diff --git a/modules/agent/api/init.go b/modules/agent/api/init.go index bcefa402..254d25f7 100644 --- a/modules/agent/api/init.go +++ b/modules/agent/api/init.go @@ -4,35 +4,32 @@ package api -import ( - "infini.sh/framework/core/api" - "infini.sh/framework/core/api/rbac/enum" -) - func Init() { - handler := APIHandler{} - api.HandleAPIMethod(api.POST, "/agent/instance", handler.createInstance) - api.HandleAPIMethod(api.GET, "/agent/instance/_search", handler.RequirePermission(handler.searchInstance, enum.PermissionAgentInstanceRead)) - api.HandleAPIMethod(api.GET, "/agent/instance/:instance_id", handler.getInstance) - api.HandleAPIMethod(api.PUT, "/agent/instance/:instance_id", handler.updateInstance) - api.HandleAPIMethod(api.DELETE, "/agent/instance/:instance_id", handler.RequirePermission(handler.deleteInstance, enum.PermissionAgentInstanceWrite)) - api.HandleAPIMethod(api.POST, "/agent/instance/_stats", handler.RequirePermission(handler.getInstanceStats, enum.PermissionAgentInstanceRead)) - api.HandleAPIMethod(api.GET, "/agent/log/node/:node_id/files", handler.RequirePermission(handler.getLogFilesByNode, enum.PermissionAgentInstanceRead)) - api.HandleAPIMethod(api.POST, "/agent/log/node/:node_id/_scroll", handler.RequirePermission(handler.getLogFileContent, enum.PermissionAgentInstanceRead)) - api.HandleAPIMethod(api.GET, "/agent/instance/:instance_id/_nodes", handler.RequirePermission(handler.getESNodesInfo, enum.PermissionAgentInstanceRead)) - api.HandleAPIMethod(api.POST, "/agent/instance/:instance_id/_nodes/_refresh", handler.RequirePermission(handler.refreshESNodesInfo, enum.PermissionAgentInstanceWrite)) - api.HandleAPIMethod(api.POST, "/agent/instance/:instance_id/node/_auth", handler.RequirePermission(handler.authESNode, enum.PermissionAgentInstanceWrite)) - api.HandleAPIMethod(api.DELETE, "/agent/instance/:instance_id/_nodes", handler.RequirePermission(handler.deleteESNode, enum.PermissionAgentInstanceWrite)) - api.HandleAPIMethod(api.POST, "/agent/instance/:instance_id/node/_associate", handler.RequirePermission(handler.associateESNode, enum.PermissionAgentInstanceWrite)) - api.HandleAPIMethod(api.POST, "/agent/instance/try_connect", handler.RequireLogin(handler.tryConnect)) - api.HandleAPIMethod(api.POST, "/agent/auto_associate", handler.RequirePermission(handler.autoAssociateESNode, enum.PermissionAgentInstanceWrite)) + //handler := APIHandler{} + //api.HandleAPIMethod(api.POST, "/instance", handler.registerInstance) //new - api.HandleAPIMethod(api.POST, "/host/_enroll", handler.enrollHost) - api.HandleAPIMethod(api.GET, "/host/:host_id/agent/info",handler.GetHostAgentInfo) - api.HandleAPIMethod(api.GET, "/host/:host_id/processes",handler.GetHostElasticProcess) - api.HandleAPIMethod(api.DELETE, "/host/:host_id",handler.deleteHost) - - - api.HandleAPIMethod(api.POST, "/agent/install_command", handler.RequireLogin(handler.generateInstallCommand)) - api.HandleAPIMethod(api.GET, "/agent/install.sh", handler.getInstallScript) + //api.HandleAPIMethod(api.POST, "/agent/instance", handler.registerInstance) + //api.HandleAPIMethod(api.GET, "/agent/instance/_search", handler.RequirePermission(handler.searchInstance, enum.PermissionAgentInstanceRead)) + //api.HandleAPIMethod(api.GET, "/agent/instance/:instance_id", handler.getInstance) + //api.HandleAPIMethod(api.PUT, "/agent/instance/:instance_id", handler.updateInstance) + //api.HandleAPIMethod(api.DELETE, "/agent/instance/:instance_id", handler.RequirePermission(handler.deleteInstance, enum.PermissionAgentInstanceWrite)) + //api.HandleAPIMethod(api.POST, "/agent/instance/_stats", handler.RequirePermission(handler.getInstanceStats, enum.PermissionAgentInstanceRead)) + //api.HandleAPIMethod(api.GET, "/agent/log/node/:node_id/files", handler.RequirePermission(handler.getLogFilesByNode, enum.PermissionAgentInstanceRead)) + //api.HandleAPIMethod(api.POST, "/agent/log/node/:node_id/_scroll", handler.RequirePermission(handler.getLogFileContent, enum.PermissionAgentInstanceRead)) + //api.HandleAPIMethod(api.GET, "/agent/instance/:instance_id/_nodes", handler.RequirePermission(handler.getESNodesInfo, enum.PermissionAgentInstanceRead)) + //api.HandleAPIMethod(api.POST, "/agent/instance/:instance_id/_nodes/_refresh", handler.RequirePermission(handler.refreshESNodesInfo, enum.PermissionAgentInstanceWrite)) + //api.HandleAPIMethod(api.POST, "/agent/instance/:instance_id/node/_auth", handler.RequirePermission(handler.authESNode, enum.PermissionAgentInstanceWrite)) + //api.HandleAPIMethod(api.DELETE, "/agent/instance/:instance_id/_nodes", handler.RequirePermission(handler.deleteESNode, enum.PermissionAgentInstanceWrite)) + //api.HandleAPIMethod(api.POST, "/agent/instance/:instance_id/node/_associate", handler.RequirePermission(handler.associateESNode, enum.PermissionAgentInstanceWrite)) + //api.HandleAPIMethod(api.POST, "/agent/instance/try_connect", handler.RequireLogin(handler.tryConnect)) + //api.HandleAPIMethod(api.POST, "/agent/auto_associate", handler.RequirePermission(handler.autoAssociateESNode, enum.PermissionAgentInstanceWrite)) + // + //api.HandleAPIMethod(api.POST, "/host/_enroll", handler.enrollHost) + //api.HandleAPIMethod(api.GET, "/host/:host_id/agent/info",handler.GetHostAgentInfo) + //api.HandleAPIMethod(api.GET, "/host/:host_id/processes",handler.GetHostElasticProcess) + //api.HandleAPIMethod(api.DELETE, "/host/:host_id",handler.deleteHost) + // + // + //api.HandleAPIMethod(api.POST, "/agent/install_command", handler.RequireLogin(handler.generateInstallCommand)) + //api.HandleAPIMethod(api.GET, "/agent/install.sh", handler.getInstallScript) } diff --git a/modules/agent/api/instance.go b/modules/agent/api/instance.go index 79bdbaf0..a6ebe0ed 100644 --- a/modules/agent/api/instance.go +++ b/modules/agent/api/instance.go @@ -5,1152 +5,9 @@ package api import ( - "context" - "fmt" - "net" - "net/http" - "net/url" - "strconv" - "strings" - "time" - - log "github.com/cihub/seelog" - "infini.sh/console/modules/agent/client" - common2 "infini.sh/console/modules/agent/common" - "infini.sh/console/modules/agent/model" - "infini.sh/console/modules/agent/state" - "infini.sh/framework/core/agent" "infini.sh/framework/core/api" - httprouter "infini.sh/framework/core/api/router" - "infini.sh/framework/core/elastic" - "infini.sh/framework/core/orm" - "infini.sh/framework/core/util" - elastic2 "infini.sh/framework/modules/elastic" - "infini.sh/framework/modules/elastic/common" ) type APIHandler struct { api.Handler } - -func (h *APIHandler) createInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - var obj = &agent.Instance{} - err := h.DecodeJSON(req, obj) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - //validate token for auto register - token := h.GetParameter(req, "token") - if token != "" { - if v, ok := tokens.Load(token); !ok { - h.WriteError(w, "token is invalid", http.StatusUnauthorized) - return - } else { - if t, ok := v.(*Token); !ok || t.CreatedAt.Add(ExpiredIn).Before(time.Now()) { - tokens.Delete(token) - h.WriteError(w, "token was expired", http.StatusUnauthorized) - return - } - } - remoteIP := util.ClientIP(req) - agCfg := common2.GetAgentConfig() - port := agCfg.Setup.Port - if port == "" { - port = "8080" - } - obj.Endpoint = fmt.Sprintf("https://%s:%s", remoteIP, port) - obj.Tags = append(obj.Tags, "mtls", "auto") - } - - //fetch more information of agent instance - res, err := client.GetClient().GetInstanceBasicInfo(context.Background(), obj.GetEndpoint()) - if err != nil { - errStr := fmt.Sprintf("get agent instance basic info error: %s", err.Error()) - h.WriteError(w, errStr, http.StatusInternalServerError) - log.Error(errStr) - return - } - if res.ID == "" { - errStr := fmt.Sprintf("got unexpected response of agent instance basic info: %s", util.MustToJSON(res)) - h.WriteError(w, errStr, http.StatusInternalServerError) - log.Error(errStr) - return - } else { - obj.ID = res.ID - obj.Version = res.Version - obj.MajorIP = res.MajorIP - obj.Host = res.Host - obj.IPS = res.IPS - if obj.Name == "" { - obj.Name = res.Name - } - } - oldInst := &agent.Instance{} - oldInst.ID = obj.ID - exists, err := orm.Get(oldInst) - - if err != nil && err != elastic2.ErrNotFound { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - if exists { - errMsg := fmt.Sprintf("agent [%s] already exists", obj.ID) - h.WriteError(w, errMsg, http.StatusInternalServerError) - log.Error(errMsg) - return - } - if token != "" { - err, result := orm.GetBy("endpoint", obj.Endpoint, oldInst) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - if len(result.Result) > 0 { - if m, ok := result.Result[0].(map[string]interface{}); ok { - if id, ok := m["id"].(string); ok { - oldInst.ID = id - err = orm.Delete(nil, oldInst) - if err != nil { - log.Error(err) - } - } - } - } - } - - obj.Status = model.StatusOnline - err = orm.Create(nil, obj) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - err = client.GetClient().SaveIngestConfig(context.Background(), obj.GetEndpoint()) - if err != nil { - log.Error(err) - } - _, err = refreshNodesInfo(obj) - if err != nil { - log.Error(err) - } - - h.WriteCreatedOKJSON(w, obj.ID) - -} - -func (h *APIHandler) getInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - id := ps.MustGetParameter("instance_id") - - obj := agent.Instance{} - obj.ID = id - - exists, err := orm.Get(&obj) - if !exists || err != nil { - h.WriteJSON(w, util.MapStr{ - "_id": id, - "found": false, - }, http.StatusNotFound) - return - } - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - - h.WriteJSON(w, util.MapStr{ - "found": true, - "_id": id, - "_source": obj, - }, 200) -} - -func (h *APIHandler) deleteInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - id := ps.MustGetParameter("instance_id") - - obj := agent.Instance{} - obj.ID = id - - exists, err := orm.Get(&obj) - if !exists || err != nil { - h.WriteJSON(w, util.MapStr{ - "_id": id, - "result": "not_found", - }, http.StatusNotFound) - return - } - - err = orm.Delete(nil, &obj) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - if sm := state.GetStateManager(); sm != nil { - sm.DeleteAgent(obj.ID) - } - queryDsl := util.MapStr{ - "query": util.MapStr{ - "term": util.MapStr{ - "agent_id": util.MapStr{ - "value": id, - }, - }, - }, - } - err = orm.DeleteBy(agent.ESNodeInfo{}, util.MustToJSONBytes(queryDsl)) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error("delete node info error: ", err) - return - } - - queryDsl = util.MapStr{ - "query": util.MapStr{ - "term": util.MapStr{ - "metadata.labels.agent_id": util.MapStr{ - "value": id, - }, - }, - }, - } - err = orm.DeleteBy(agent.Setting{}, util.MustToJSONBytes(queryDsl)) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error("delete agent settings error: ", err) - return - } - - h.WriteDeletedOKJSON(w, id) -} - -func (h *APIHandler) getInstanceStats(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - var instanceIDs = []string{} - err := h.DecodeJSON(req, &instanceIDs) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - if len(instanceIDs) == 0 { - h.WriteJSON(w, util.MapStr{}, http.StatusOK) - return - } - q := orm.Query{} - queryDSL := util.MapStr{ - "size": len(instanceIDs), - "query": util.MapStr{ - "terms": util.MapStr{ - "_id": instanceIDs, - }, - }, - } - q.RawQuery = util.MustToJSONBytes(queryDSL) - - err, res := orm.Search(&agent.Instance{}, &q) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - result := util.MapStr{} - for _, item := range res.Result { - instBytes, err := util.ToJSONBytes(item) - if err != nil { - log.Error(err) - continue - } - instance := agent.Instance{} - err = util.FromJSONBytes(instBytes, &instance) - if err != nil { - log.Error(err) - continue - } - agReq := &util.Request{ - Method: http.MethodGet, - Url: fmt.Sprintf("%s/stats", instance.GetEndpoint()), - - } - var resMap = util.MapStr{} - err = client.GetClient().DoRequest(agReq, &resMap) - - if err != nil { - log.Error(err) - result[instance.ID] = util.MapStr{} - continue - } - result[instance.ID] = resMap - } - h.WriteJSON(w, result, http.StatusOK) -} - -func (h *APIHandler) updateInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - id := ps.MustGetParameter("instance_id") - oldInst := agent.Instance{} - oldInst.ID = id - _, err := orm.Get(&oldInst) - if err != nil { - if err == elastic2.ErrNotFound { - h.WriteJSON(w, util.MapStr{ - "_id": id, - "result": "not_found", - }, http.StatusNotFound) - return - } - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - - obj := agent.Instance{} - err = h.DecodeJSON(req, &obj) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - - oldInst.Name = obj.Name - oldInst.Endpoint = obj.Endpoint - oldInst.Description = obj.Description - oldInst.Tags = obj.Tags - oldInst.BasicAuth = obj.BasicAuth - err = orm.Update(&orm.Context{ - Refresh: "wait_for", - }, &oldInst) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - - h.WriteJSON(w, util.MapStr{ - "_id": obj.ID, - "result": "updated", - }, 200) -} - -func (h *APIHandler) searchInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - - var ( - keyword = h.GetParameterOrDefault(req, "keyword", "") - //queryDSL = `{"query":{"bool":{"must":[%s]}}, "size": %d, "from": %d}` - strSize = h.GetParameterOrDefault(req, "size", "20") - strFrom = h.GetParameterOrDefault(req, "from", "0") - ) - - var ( - mustQ []interface{} - ) - - if keyword != "" { - mustQ = append(mustQ, util.MapStr{ - "query_string": util.MapStr{ - "default_field": "*", - "query": keyword, - }, - }) - } - size, _ := strconv.Atoi(strSize) - if size <= 0 { - size = 20 - } - from, _ := strconv.Atoi(strFrom) - if from < 0 { - from = 0 - } - - queryDSL := util.MapStr{ - "size": size, - "from": from, - } - if len(mustQ) > 0 { - queryDSL["query"] = util.MapStr{ - "bool": util.MapStr{ - "must": mustQ, - }, - } - } - - q := orm.Query{} - q.RawQuery = util.MustToJSONBytes(queryDSL) - - err, res := orm.Search(&agent.Instance{}, &q) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - h.Write(w, res.Raw) -} - -func (h *APIHandler) getESNodesInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - id := ps.MustGetParameter("instance_id") - obj := agent.Instance{} - obj.ID = id - exists, err := orm.Get(&obj) - if !exists || err != nil { - h.WriteJSON(w, util.MapStr{ - "_id": id, - "found": false, - }, http.StatusNotFound) - return - } - nodes, err := refreshNodesInfo(&obj) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - var nodeUUIDs []string - for _, node := range nodes { - if node.NodeUUID != "" { - nodeUUIDs = append(nodeUUIDs, node.NodeUUID) - } - } - if len(nodeUUIDs) == 0 { - h.WriteJSON(w, nodes, http.StatusOK) - return - } - query := util.MapStr{ - "size": len(nodeUUIDs), - "query": util.MapStr{ - "terms": util.MapStr{ - "metadata.node_id": nodeUUIDs, - }, - }, - "collapse": util.MapStr{ - "field": "metadata.node_id", - }, - } - q := orm.Query{ - RawQuery: util.MustToJSONBytes(query), - } - err, result := orm.Search(elastic.NodeConfig{}, &q) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - idToAddresses := map[string]string{} - for _, row := range result.Result { - if rowM, ok := row.(map[string]interface{}); ok { - nodeUUID, _ := util.MapStr(rowM).GetValue("metadata.node_id") - transportAddr, _ := util.MapStr(rowM).GetValue("metadata.labels.transport_address") - if v, ok := nodeUUID.(string); ok { - idToAddresses[v] = transportAddr.(string) - } - } - } - var nNodes []tempNode - for _, node := range nodes { - nNode := tempNode{ - ESNodeInfo: node, - } - if node.NodeUUID != "" { - if addr, ok := idToAddresses[node.NodeUUID]; ok { - nNode.TransportAddress = addr - } - } - nNodes = append(nNodes, nNode) - } - - - h.WriteJSON(w, nNodes, http.StatusOK) -} -type tempNode struct { - agent.ESNodeInfo - TransportAddress string `json:"transport_address"` -} - -func (h *APIHandler) refreshESNodesInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - id := ps.MustGetParameter("instance_id") - obj := agent.Instance{} - obj.ID = id - exists, err := orm.Get(&obj) - if !exists || err != nil { - h.WriteJSON(w, util.MapStr{ - "_id": id, - "found": false, - }, http.StatusNotFound) - return - } - _, err = refreshNodesInfo(&obj) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - h.WriteAckOKJSON(w) -} - -func (h *APIHandler) authESNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - id := ps.MustGetParameter("instance_id") - inst := agent.Instance{} - inst.ID = id - exists, err := orm.Get(&inst) - if !exists || err != nil { - h.WriteJSON(w, util.MapStr{ - "_id": id, - "found": false, - }, http.StatusNotFound) - return - } - reqBody := struct { - NodeID string `json:"node_id"` - ESConfig *elastic.ElasticsearchConfig `json:"es_config"` - }{} - err = h.DecodeJSON(req, &reqBody) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - oldNodeInfo := &agent.ESNodeInfo{ - ID: reqBody.NodeID, - } - exists, err = orm.Get(oldNodeInfo) - if !exists || err != nil { - h.WriteJSON(w, fmt.Sprintf("node [%s] of agent [%s] was not found", oldNodeInfo.ID, inst.Name), http.StatusInternalServerError) - return - } - - cfg := reqBody.ESConfig - if cfg.Endpoint == "" { - cfg.Endpoint = fmt.Sprintf("%s://%s", cfg.Schema, cfg.Host) - } - basicAuth, err := common.GetBasicAuth(cfg) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - cfg.BasicAuth = &basicAuth - nodeInfo, err := client.GetClient().AuthESNode(context.Background(), inst.GetEndpoint(), *cfg) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - host, port, err := net.SplitHostPort(nodeInfo.PublishAddress) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - if !util.StringInArray(inst.IPS, host) && !net.ParseIP(host).IsLoopback() { - h.WriteError(w, fmt.Sprintf("got node host %s not match any ip of %v", host, inst.IPS), http.StatusInternalServerError) - return - } - if oldNodeInfo.HttpPort != port { - h.WriteError(w, fmt.Sprintf("port mismatch, got: %s，expected: %s", port, oldNodeInfo.HttpPort), http.StatusInternalServerError) - return - } - if oldNodeInfo.ProcessInfo.PID != nodeInfo.ProcessInfo.PID { - h.WriteError(w, fmt.Sprintf("process id mismatch, got: %d，expected: %d", nodeInfo.ProcessInfo.PID, oldNodeInfo.ProcessInfo.PID), http.StatusInternalServerError) - return - } - - nodeInfo.ID = oldNodeInfo.ID - nodeInfo.AgentID = inst.ID - err = orm.Save(nil, nodeInfo) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - h.WriteJSON(w, nodeInfo, http.StatusOK) -} - -func (h *APIHandler) associateESNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - instID := ps.MustGetParameter("instance_id") - reqBody := struct { - ID string `json:"id"` - ClusterID string `json:"cluster_id"` - }{} - err := h.DecodeJSON(req, &reqBody) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - node := agent.ESNodeInfo{ - ID: reqBody.ID, - } - _, err = orm.Get(&node) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - if node.AgentID != instID { - errStr := fmt.Sprintf("agent id not match: %s, %s", node.AgentID, instID) - log.Error(errStr) - h.WriteError(w, errStr, http.StatusInternalServerError) - return - } - node.ClusterID = reqBody.ClusterID - err = orm.Save(&orm.Context{ - Refresh: "wait_for", - }, node) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - settings, err := common2.GetAgentSettings(instID, 0) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - setting := pickAgentSettings(settings, node) - if setting == nil { - setting, err = getAgentTaskSetting(instID, node) - if err != nil { - log.Error("get agent task setting error: ", err) - } - err = orm.Create(nil, setting) - if err != nil { - log.Error("save agent task setting error: ", err) - } - } - h.WriteAckOKJSON(w) -} - -func (h *APIHandler) autoAssociateESNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - reqBody := struct { - ClusterIDs []string `json:"cluster_ids"` - }{} - err := h.DecodeJSON(req, &reqBody) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - // query not associated nodes info - nodesM, err := getUnAssociateNodes() - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - if len(nodesM) == 0 { - h.WriteAckOKJSON(w) - return - } - agentIds := make([]string, 0, len(nodesM)) - for agentID := range nodesM { - agentIds = append(agentIds, agentID) - } - agents, err := getAgentByIds(agentIds) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - for _, clusterID := range reqBody.ClusterIDs { - // query cluster basicauth - cfg := elastic.GetConfig(clusterID) - basicAuth, err := common.GetBasicAuth(cfg) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - taskSetting, err := getSettingsByClusterID(cfg.ID) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - for agentID, nodes := range nodesM { - var ( - inst *agent.Instance - ok bool - ) - if inst, ok = agents[agentID]; !ok { - log.Warnf("agent [%v] was not found", agentID) - continue - } - settings, err := common2.GetAgentSettings(agentID, 0) - if err != nil { - log.Error(err) - continue - } - for _, node := range nodes { - host := node.PublishAddress - var endpoint string - if strings.HasPrefix(host, "::") { - instURL, err := url.Parse(inst.Endpoint) - if err != nil { - log.Error(err) - continue - } - host = instURL.Hostname() - endpoint = fmt.Sprintf("%s://%s:%s", node.Schema, host, node.HttpPort) - } else { - endpoint = fmt.Sprintf("%s://%s", node.Schema, host) - } - escfg := elastic.ElasticsearchConfig{ - Endpoint: endpoint, - BasicAuth: &basicAuth, - } - nodeInfo, err := client.GetClient().AuthESNode(context.Background(), inst.GetEndpoint(), escfg) - if err != nil { - log.Warn(err) - continue - } - //matched - if nodeInfo.ClusterUuid == cfg.ClusterUUID { - //update node info - nodeInfo.ID = node.ID - nodeInfo.AgentID = inst.ID - nodeInfo.ClusterID = cfg.ID - err = orm.Save(nil, nodeInfo) - if err != nil { - log.Error(err) - continue - } - setting := pickAgentSettings(settings, node) - if setting == nil { - tsetting := model.TaskSetting{ - NodeStats: &model.NodeStatsTask{ - Enabled: true, - }, - Logs: &model.LogsTask{ - Enabled: true, - LogsPath: nodeInfo.Path.Logs, - }, - } - if taskSetting.IndexStats != nil { - tsetting.IndexStats = taskSetting.IndexStats - taskSetting.IndexStats = nil - } - if taskSetting.ClusterHealth != nil { - tsetting.ClusterHealth = taskSetting.ClusterHealth - taskSetting.ClusterHealth = nil - } - if taskSetting.ClusterStats != nil { - tsetting.ClusterStats = taskSetting.ClusterStats - taskSetting.ClusterStats = nil - } - setting = &agent.Setting{ - Metadata: agent.SettingsMetadata{ - Category: "agent", - Name: "task", - Labels: util.MapStr{ - "agent_id": agentID, - "cluster_uuid": nodeInfo.ClusterUuid, - "cluster_id": nodeInfo.ClusterID, - "node_uuid": nodeInfo.NodeUUID, - "endpoint": fmt.Sprintf("%s://%s", nodeInfo.Schema, nodeInfo.PublishAddress), - }, - }, - Payload: util.MapStr{ - "task": tsetting, - }, - } - err = orm.Create(nil, setting) - if err != nil { - log.Error("save agent task setting error: ", err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - } - } - } - - } - } - h.WriteAckOKJSON(w) -} - -func getAgentByIds(agentIDs []string)(map[string]*agent.Instance, error){ - query := util.MapStr{ - "size": len(agentIDs), - "query": util.MapStr{ - "terms": util.MapStr{ - "id": agentIDs, - }, - }, - } - q := orm.Query{ - RawQuery: util.MustToJSONBytes(query), - } - err, result := orm.Search(agent.Instance{}, &q) - if err != nil { - return nil, err - } - agents := map[string]*agent.Instance{} - for _, row := range result.Result { - inst := agent.Instance{} - buf := util.MustToJSONBytes(row) - util.MustFromJSONBytes(buf, &inst) - agents[inst.ID] = &inst - } - return agents, nil -} - -func (h *APIHandler) deleteESNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - id := ps.MustGetParameter("instance_id") - nodeIDs := []string{} - err := h.DecodeJSON(req, &nodeIDs) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - if len(nodeIDs) > 0 { - q := util.MapStr{ - "query": util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "terms": util.MapStr{ - "id": nodeIDs, - }, - }, - { - "term": util.MapStr{ - "agent_id": util.MapStr{ - "value": id, - }, - }, - }, - }, - }, - }, - } - err = orm.DeleteBy(agent.ESNodeInfo{}, util.MustToJSONBytes(q)) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - q = util.MapStr{ - "query": util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "terms": util.MapStr{ - "metadata.labels.node_uuid": nodeIDs, - }, - }, - { - "term": util.MapStr{ - "metadata.labels.agent_id": util.MapStr{ - "value": id, - }, - }, - }, - }, - }, - }, - } - } - h.WriteAckOKJSON(w) -} - -func (h *APIHandler) tryConnect(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - var reqBody = struct { - Endpoint string `json:"endpoint"` - BasicAuth agent.BasicAuth - }{} - err := h.DecodeJSON(req, &reqBody) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - connectRes, err := client.GetClient().GetInstanceBasicInfo(context.Background(), reqBody.Endpoint) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - h.WriteJSON(w, connectRes, http.StatusOK) -} - -func refreshNodesInfo(inst *agent.Instance) ([]agent.ESNodeInfo, error) { - oldNodesInfo, err := getNodesInfoFromES(inst.ID) - if err != nil { - return nil, fmt.Errorf("get elasticsearch nodes info from es error: %w", err) - } - nodesInfo, err := client.GetClient().GetElasticsearchNodes(context.Background(), inst.GetEndpoint()) - if err != nil { - log.Errorf("get elasticsearch nodes error: %v", err) - //return nodes info from es after failed to get nodes info from agent - var nodes = []agent.ESNodeInfo{} - for _, nodeInfo := range oldNodesInfo { - nodes = append(nodes, *nodeInfo) - } - return nodes, nil - } - - oldPids := map[int]struct{}{} - var resultNodes []agent.ESNodeInfo - if err != nil { - return nil, err - } - for _, node := range nodesInfo { - oldNode := getNodeByPidOrUUID(oldNodesInfo, node.ProcessInfo.PID, node.NodeUUID, node.HttpPort) - node.AgentID = inst.ID - if oldNode != nil { - node.ID = oldNode.ID - //keep old validate info - if node.ClusterUuid == "" && oldNode.ClusterUuid != "" { - node = *oldNode - } - oldPids[oldNode.ProcessInfo.PID] = struct{}{} - } else { - node.ID = util.GetUUID() - } - if node.ClusterUuid != "" { - if oldNode != nil && oldNode.ClusterID != "" { - node.ClusterID = oldNode.ClusterID - } - } - - node.Status = "online" - err = orm.Save(nil, node) - if err != nil { - log.Error("save node info error: ", err) - } - resultNodes = append(resultNodes, node) - } - for k, node := range oldNodesInfo { - if _, ok := oldPids[k]; !ok { - //auto delete not associated cluster - if node.ClusterID == "" { - log.Info("delete node with pid: ", node.ProcessInfo.PID) - err = orm.Delete(nil, node) - if err != nil { - log.Error("delete node info error: ", err) - } - continue - } - node.Status = "offline" - err = orm.Save(nil, node) - if err != nil { - log.Error("save node info error: ", err) - } - resultNodes = append(resultNodes, *node) - } - } - return resultNodes, nil -} - -func getNodeByPidOrUUID(nodes map[int]*agent.ESNodeInfo, pid int, uuid string, port string) *agent.ESNodeInfo { - if nodes[pid] != nil { - return nodes[pid] - } - for _, node := range nodes { - if node.NodeUUID != "" && node.NodeUUID == uuid { - return node - } - } - return nil -} - -func getNodesInfoFromES(agentID string) (map[int]*agent.ESNodeInfo, error) { - query := util.MapStr{ - "size": 1000, - "query": util.MapStr{ - "term": util.MapStr{ - "agent_id": util.MapStr{ - "value": agentID, - }, - }, - }, - } - q := orm.Query{ - RawQuery: util.MustToJSONBytes(query), - } - - err, result := orm.Search(agent.ESNodeInfo{}, &q) - if err != nil { - return nil, err - } - nodesInfo := map[int]*agent.ESNodeInfo{} - for _, row := range result.Result { - node := agent.ESNodeInfo{} - buf := util.MustToJSONBytes(row) - util.MustFromJSONBytes(buf, &node) - nodesInfo[node.ProcessInfo.PID] = &node - } - return nodesInfo, nil -} - -func getUnAssociateNodes() (map[string][]agent.ESNodeInfo, error){ - query := util.MapStr{ - "size": 3000, - "query": util.MapStr{ - "bool": util.MapStr{ - "must_not": []util.MapStr{ - { - "exists": util.MapStr{ - "field": "cluster_id", - }, - }, - }, - }, - }, - } - q := orm.Query{ - RawQuery: util.MustToJSONBytes(query), - } - - err, result := orm.Search(agent.ESNodeInfo{}, &q) - if err != nil { - return nil, err - } - nodesInfo := map[string][]agent.ESNodeInfo{} - for _, row := range result.Result { - node := agent.ESNodeInfo{} - buf := util.MustToJSONBytes(row) - util.MustFromJSONBytes(buf, &node) - nodesInfo[node.AgentID] = append(nodesInfo[node.AgentID], node) - } - return nodesInfo, nil -} - -func pickAgentSettings(settings []agent.Setting, nodeInfo agent.ESNodeInfo) *agent.Setting { - for _, setting := range settings { - if setting.Metadata.Labels["node_uuid"] == nodeInfo.NodeUUID { - return &setting - } - } - return nil -} - -func getAgentTaskSetting(agentID string, node agent.ESNodeInfo) (*agent.Setting, error) { - taskSetting, err := getSettingsByClusterID(node.ClusterID) - if err != nil { - return nil, err - } - taskSetting.Logs = &model.LogsTask{ - Enabled: true, - LogsPath: node.Path.Logs, - } - return &agent.Setting{ - Metadata: agent.SettingsMetadata{ - Category: "agent", - Name: "task", - Labels: util.MapStr{ - "agent_id": agentID, - "cluster_uuid": node.ClusterUuid, - "cluster_id": node.ClusterID, - "node_uuid": node.NodeUUID, - "endpoint": fmt.Sprintf("%s://%s", node.Schema, node.PublishAddress), - }, - }, - Payload: util.MapStr{ - "task": taskSetting, - }, - }, nil -} - -// getSettingsByClusterID query agent task settings with cluster id -func getSettingsByClusterID(clusterID string) (*model.TaskSetting, error) { - err, result := querySettingsByClusterID(clusterID) - if err != nil { - return nil, err - } - - setting := &model.TaskSetting{ - NodeStats: &model.NodeStatsTask{ - Enabled: true, - }, - } - var ( - clusterStats = true - indexStats = true - clusterHealth = true - ) - keys := []string{"payload.task.cluster_stats.enabled", "payload.task.cluster_health.enabled", "payload.task.index_stats.enabled"} - for _, row := range result.Result { - if v, ok := row.(map[string]interface{}); ok { - vm := util.MapStr(v) - for _, key := range keys { - tv, _ := vm.GetValue(key) - if tv == true { - switch key { - case "payload.task.cluster_stats.enabled": - clusterStats = false - case "payload.task.index_stats.enabled": - indexStats = false - case "payload.task.cluster_health.enabled": - clusterHealth = false - } - } - } - } - } - if clusterStats { - setting.ClusterStats = &model.ClusterStatsTask{ - Enabled: true, - } - } - if indexStats { - setting.IndexStats = &model.IndexStatsTask{ - Enabled: true, - } - } - if clusterHealth { - setting.ClusterHealth = &model.ClusterHealthTask{ - Enabled: true, - } - } - return setting, nil -} - -func querySettingsByClusterID(clusterID string)(error, orm.Result){ - queryDsl := util.MapStr{ - "size": 500, - "query": util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "term": util.MapStr{ - "metadata.labels.cluster_id": util.MapStr{ - "value": clusterID, - }, - }, - }, - }, - "minimum_should_match": 1, - "should": []util.MapStr{ - { - "term": util.MapStr{ - "payload.task.cluster_health.enabled": util.MapStr{ - "value": true, - }, - }, - }, - { - "term": util.MapStr{ - "payload.task.cluster_stats.enabled": util.MapStr{ - "value": true, - }, - }, - }, - { - "term": util.MapStr{ - "payload.task.index_stats.enabled": util.MapStr{ - "value": true, - }, - }, - }, - }, - }, - }, - } - q := orm.Query{ - RawQuery: util.MustToJSONBytes(queryDsl), - } - return orm.Search(agent.Setting{}, &q) -} \ No newline at end of file diff --git a/modules/agent/api/log.go b/modules/agent/api/log.go index a50d3e8f..e4bb6341 100644 --- a/modules/agent/api/log.go +++ b/modules/agent/api/log.go @@ -9,8 +9,8 @@ import ( log "github.com/cihub/seelog" "infini.sh/console/modules/agent/client" "infini.sh/console/modules/agent/state" - "infini.sh/framework/core/agent" httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/model" "infini.sh/framework/core/orm" "infini.sh/framework/core/util" "net/http" @@ -80,7 +80,7 @@ func (h *APIHandler) getLogFileContent(w http.ResponseWriter, req *http.Request, h.WriteJSON(w, res, http.StatusOK) } -func getAgentByNodeID(nodeID string) (*agent.Instance, *agent.ESNodeInfo, error){ +func getAgentByNodeID(nodeID string) (*model.Instance, *model.ESNodeInfo, error){ queryDsl := util.MapStr{ "size":1, "query": util.MapStr{ @@ -101,24 +101,24 @@ func getAgentByNodeID(nodeID string) (*agent.Instance, *agent.ESNodeInfo, error) q := &orm.Query{ RawQuery: util.MustToJSONBytes(queryDsl), } - err, result := orm.Search(agent.ESNodeInfo{}, q) + err, result := orm.Search(model.ESNodeInfo{}, q) if err != nil { return nil,nil, err } if len(result.Result) > 0 { buf := util.MustToJSONBytes(result.Result[0]) - node := &agent.ESNodeInfo{} - err = util.FromJSONBytes(buf, node) - inst := &agent.Instance{} - inst.ID = node.AgentID + v := &model.ESNodeInfo{} + err = util.FromJSONBytes(buf, v) + inst := &model.Instance{} + inst.ID = v.AgentID _, err = orm.Get(inst) if err != nil { - return nil, node, err + return nil, v, err } if inst.Name == "" { - return nil, node, nil + return nil, v, nil } - return inst, node, nil + return inst, v, nil } return nil, nil, nil } diff --git a/modules/agent/api/setup.go b/modules/agent/api/setup.go deleted file mode 100644 index 5c1c5253..00000000 --- a/modules/agent/api/setup.go +++ /dev/null @@ -1,155 +0,0 @@ -/* Copyright © INFINI Ltd. All rights reserved. - * Web: https://infinilabs.com - * Email: hello#infini.ltd */ - -package api - -import ( - "fmt" - log "github.com/cihub/seelog" - "github.com/valyala/fasttemplate" - "infini.sh/console/modules/agent/common" - "infini.sh/framework/core/api/rbac" - httprouter "infini.sh/framework/core/api/router" - "infini.sh/framework/core/global" - "infini.sh/framework/core/util" - "os" - - "net/http" - "path" - "strings" - "sync" - "time" -) - -var tokens = sync.Map{} -type Token struct { - CreatedAt time.Time - UserID string -} - -const ExpiredIn = time.Millisecond * 1000 * 60 * 60 -func (h *APIHandler) generateInstallCommand(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - claims, ok := req.Context().Value("user").(*rbac.UserClaims) - if !ok { - h.WriteError(w, "user not found", http.StatusInternalServerError) - return - } - agCfg := common.GetAgentConfig() - if agCfg == nil || agCfg.Setup == nil { - h.WriteError(w, "agent setup config was not found, please configure in the configuration file first", http.StatusInternalServerError) - return - } - var ( - t *Token - tokenStr string - ) - tokens.Range(func(key, value any) bool { - if v, ok := value.(*Token); ok && claims.UserId == v.UserID { - t = v - tokenStr = key.(string) - return false - } - return true - }) - - if t == nil { - tokenStr = util.GetUUID() - t = &Token{ - CreatedAt: time.Now(), - UserID: claims.UserId, - } - }else{ - if t.CreatedAt.Add(ExpiredIn).Before(time.Now()){ - tokens.Delete(tokenStr) - tokenStr = util.GetUUID() - t = &Token{ - CreatedAt: time.Now(), - UserID: claims.UserId, - } - }else{ - t.CreatedAt = time.Now() - } - } - tokens.Store(tokenStr, t) - consoleEndpoint := agCfg.Setup.ConsoleEndpoint - if consoleEndpoint == "" { - consoleEndpoint = getDefaultConsoleEndpoint(req) - } - - - h.WriteJSON(w, util.MapStr{ - "script": fmt.Sprintf(`curl -sSL %s/agent/install.sh?token=%s |sudo bash -s -- -u %s -v %s -t /opt/agent`, consoleEndpoint, tokenStr, agCfg.Setup.DownloadURL, agCfg.Setup.Version), - //"script": fmt.Sprintf(`sudo BASE_URL="%s" AGENT_VER="%s" INSTALL_PATH="/opt" bash -c "$(curl -L '%s/agent/install.sh?token=%s')"`, agCfg.Setup.DownloadURL, agCfg.Setup.Version, consoleEndpoint, tokenStr), - "token": tokenStr, - "expired_at": t.CreatedAt.Add(ExpiredIn), - }, http.StatusOK) -} - -func getDefaultConsoleEndpoint(req *http.Request) string{ - scheme := "http" - if req.TLS != nil { - scheme = "https" - } - return fmt.Sprintf("%s://%s", scheme, req.Host) -} - -func (h *APIHandler) getInstallScript(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - tokenStr := h.GetParameter(req, "token") - if strings.TrimSpace(tokenStr) == "" { - h.WriteError(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) - return - } - if v, ok := tokens.Load(tokenStr); !ok { - h.WriteError(w, "token is invalid", http.StatusUnauthorized) - return - }else{ - if t, ok := v.(*Token); !ok || t.CreatedAt.Add(ExpiredIn).Before(time.Now()) { - tokens.Delete(tokenStr) - h.WriteError(w, "token was expired", http.StatusUnauthorized) - return - } - } - agCfg := common.GetAgentConfig() - caCert, clientCertPEM, clientKeyPEM, err := common.GenerateServerCert(agCfg.Setup.CACertFile, agCfg.Setup.CAKeyFile) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - scriptTplPath := path.Join(global.Env().GetConfigDir(), "install_agent.tpl") - buf, err := os.ReadFile(scriptTplPath) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - tpl := fasttemplate.New(string(buf), "{{", "}}") - downloadURL := agCfg.Setup.DownloadURL - if downloadURL == "" { - downloadURL = "https://release.infinilabs.com/agent/stable/" - } - port := agCfg.Setup.Port - if port == "" { - port = "8080" - } - consoleEndpoint := agCfg.Setup.ConsoleEndpoint - if consoleEndpoint == "" { - consoleEndpoint = getDefaultConsoleEndpoint(req) - } - _, err = tpl.Execute(w, map[string]interface{}{ - "base_url": agCfg.Setup.DownloadURL, - "agent_version": agCfg.Setup.Version, - "console_endpoint": consoleEndpoint, - "client_crt": clientCertPEM, - "client_key": clientKeyPEM, - "ca_crt": caCert, - "port": port, - "token": tokenStr, - }) - if err != nil { - log.Error(err) - } -} - diff --git a/modules/agent/client/client.go b/modules/agent/client/client.go index 586dc67f..7f26092d 100644 --- a/modules/agent/client/client.go +++ b/modules/agent/client/client.go @@ -8,7 +8,7 @@ import ( "context" "fmt" "infini.sh/console/modules/agent/common" - "infini.sh/framework/core/agent" + "infini.sh/framework/core/model" "infini.sh/framework/core/elastic" "infini.sh/framework/core/host" "infini.sh/framework/core/util" @@ -32,10 +32,10 @@ type ClientAPI interface { GetElasticProcess(ctx context.Context, agentBaseURL string, agentID string)(interface{}, error) GetElasticLogFiles(ctx context.Context, agentBaseURL string, logsPath string)(interface{}, error) GetElasticLogFileContent(ctx context.Context, agentBaseURL string, body interface{})(interface{}, error) - GetInstanceBasicInfo(ctx context.Context, agentBaseURL string) (*agent.Instance, error) + GetInstanceBasicInfo(ctx context.Context, agentBaseURL string) (*model.Instance, error) RegisterElasticsearch(ctx context.Context, agentBaseURL string, cfgs []elastic.ElasticsearchConfig) error - GetElasticsearchNodes(ctx context.Context, agentBaseURL string) ([]agent.ESNodeInfo, error) - AuthESNode(ctx context.Context, agentBaseURL string, cfg elastic.ElasticsearchConfig) (*agent.ESNodeInfo, error) + GetElasticsearchNodes(ctx context.Context, agentBaseURL string) ([]model.ESNodeInfo, error) + AuthESNode(ctx context.Context, agentBaseURL string, cfg elastic.ElasticsearchConfig) (*model.ESNodeInfo, error) CreatePipeline(ctx context.Context, agentBaseURL string, body []byte) error DeletePipeline(ctx context.Context, agentBaseURL, pipelineID string) error SetKeystoreValue(ctx context.Context, agentBaseURL string, key, value string) error @@ -49,28 +49,30 @@ type Client struct { } -func (client *Client) GetHostInfo(ctx context.Context, agentBaseURL string) (*host.HostInfo, error) { - req := &util.Request{ - Method: http.MethodGet, - Url: fmt.Sprintf("%s/agent/host/_basic", agentBaseURL), - Context: ctx, - } - resBody := struct { - Success bool `json:"success"` - Error string `json:"error"` - HostInfo *host.HostInfo `json:"result"` - }{} - err := client.DoRequest(req, &resBody) - if err != nil { - return nil, err - } - if resBody.Success != true { - return nil, fmt.Errorf("enroll error from client: %v", resBody.Error) - } - return resBody.HostInfo, nil -} +//func (client *Client) GetHostInfo(ctx context.Context, agentBaseURL string) (*host.HostInfo, error) { +// req := &util.Request{ +// Method: http.MethodGet, +// Url: fmt.Sprintf("%s/agent/host/_basic", agentBaseURL), +// Context: ctx, +// } +// resBody := struct { +// Success bool `json:"success"` +// Error string `json:"error"` +// HostInfo *host.HostInfo `json:"result"` +// }{} +// err := client.DoRequest(req, &resBody) +// if err != nil { +// return nil, err +// } +// if resBody.Success != true { +// return nil, fmt.Errorf("enroll error from client: %v", resBody.Error) +// } +// return resBody.HostInfo, nil +//} +//TODO func (client *Client) GetElasticProcess(ctx context.Context, agentBaseURL string, agentID string)(interface{}, error) { + panic("implement me") req := &util.Request{ Method: http.MethodGet, Url: fmt.Sprintf("%s/elasticsearch/%s/process/_elastic", agentBaseURL, agentID), @@ -88,6 +90,9 @@ func (client *Client) GetElasticProcess(ctx context.Context, agentBaseURL string } func (client *Client) GetElasticLogFiles(ctx context.Context, agentBaseURL string, logsPath string)(interface{}, error) { + panic("implement me") + + reqBody := util.MustToJSONBytes(util.MapStr{ "logs_path": logsPath, }) @@ -133,13 +138,13 @@ func (client *Client) GetElasticLogFileContent(ctx context.Context, agentBaseURL } , nil } -func (client *Client) GetInstanceBasicInfo(ctx context.Context, agentBaseURL string) (*agent.Instance, error){ +func (client *Client) GetInstanceBasicInfo(ctx context.Context, agentBaseURL string) (*model.Instance, error){ req := &util.Request{ Method: http.MethodGet, - Url: fmt.Sprintf("%s/agent/_info", agentBaseURL ), + Url: fmt.Sprintf("%s/_info", agentBaseURL ), Context: ctx, } - resBody := &agent.Instance{} + resBody := &model.Instance{} err := client.DoRequest(req, &resBody) return resBody, err } @@ -166,13 +171,13 @@ func (client *Client) RegisterElasticsearch(ctx context.Context, agentBaseURL st return nil } -func (client *Client) GetElasticsearchNodes(ctx context.Context, agentBaseURL string) ([]agent.ESNodeInfo, error) { +func (client *Client) GetElasticsearchNodes(ctx context.Context, agentBaseURL string) ([]model.ESNodeInfo, error) { req := &util.Request{ Method: http.MethodGet, - Url: fmt.Sprintf("%s/elasticsearch/_nodes", agentBaseURL ), + Url: fmt.Sprintf("%s/elasticsearch/nodes/_discovery", agentBaseURL ), Context: ctx, } - resBody := []agent.ESNodeInfo{} + resBody := []model.ESNodeInfo{} err := client.DoRequest(req, &resBody) if err != nil { return nil, err @@ -181,7 +186,7 @@ func (client *Client) GetElasticsearchNodes(ctx context.Context, agentBaseURL st return resBody, nil } -func (client *Client) AuthESNode(ctx context.Context, agentBaseURL string, cfg elastic.ElasticsearchConfig) (*agent.ESNodeInfo, error) { +func (client *Client) AuthESNode(ctx context.Context, agentBaseURL string, cfg elastic.ElasticsearchConfig) (*model.ESNodeInfo, error) { reqBody, err := util.ToJSONBytes(cfg) if err != nil { return nil, err @@ -192,7 +197,7 @@ func (client *Client) AuthESNode(ctx context.Context, agentBaseURL string, cfg e Context: ctx, Body: reqBody, } - resBody := &agent.ESNodeInfo{} + resBody := &model.ESNodeInfo{} err = client.DoRequest(req, resBody) if err != nil { return nil, err @@ -227,22 +232,22 @@ func (client *Client) SetKeystoreValue(ctx context.Context, agentBaseURL string, } req := &util.Request{ Method: http.MethodPost, - Url: fmt.Sprintf("%s/_framework/keystore", agentBaseURL), + Url: fmt.Sprintf("%s/keystore", agentBaseURL), Context: ctx, Body: util.MustToJSONBytes(body), } return client.DoRequest(req, nil) } -func (client *Client) SaveDynamicConfig(ctx context.Context, agentBaseURL string, name, content string) error{ +func (client *Client) SaveDynamicConfig(ctx context.Context, agentBaseURL string, filename, content string) error{ body := util.MapStr{ "configs": util.MapStr{ - name: content, + filename: content, }, } req := &util.Request{ Method: http.MethodPost, - Url: fmt.Sprintf("%s/agent/config", agentBaseURL), + Url: fmt.Sprintf("%s/config/_update", agentBaseURL), Context: ctx, Body: util.MustToJSONBytes(body), } @@ -260,7 +265,7 @@ func (client *Client) SaveIngestConfig(ctx context.Context, agentBaseURL string) return fmt.Errorf("set keystore value to agent error: %w", err) } } - err = client.SaveDynamicConfig(context.Background(), agentBaseURL, "ingest", ingestCfg ) + err = client.SaveDynamicConfig(context.Background(), agentBaseURL, "ingest_variables.yml", ingestCfg ) if err != nil { return fmt.Errorf("save dynamic config to agent error: %w", err) } diff --git a/modules/agent/common/helper.go b/modules/agent/common/helper.go index ff47c752..5f75db2a 100644 --- a/modules/agent/common/helper.go +++ b/modules/agent/common/helper.go @@ -6,19 +6,19 @@ package common import ( "fmt" - "infini.sh/console/modules/agent/model" - "infini.sh/framework/core/agent" + log "github.com/cihub/seelog" + model2 "infini.sh/console/modules/agent/model" "infini.sh/framework/core/credential" "infini.sh/framework/core/elastic" "infini.sh/framework/core/event" "infini.sh/framework/core/global" + "infini.sh/framework/core/model" "infini.sh/framework/core/orm" "infini.sh/framework/core/util" - log "src/github.com/cihub/seelog" "strings" ) -func ParseAgentSettings(settings []agent.Setting)(*model.ParseAgentSettingsResult, error){ +func ParseAgentSettings(settings []model.Setting)(*model2.ParseAgentSettingsResult, error){ var clusterCfgs []elastic.ElasticsearchConfig var ( pipelines []util.MapStr @@ -58,7 +58,7 @@ func ParseAgentSettings(settings []agent.Setting)(*model.ParseAgentSettingsResul if err != nil { return nil, err } - taskSetting := model.TaskSetting{} + taskSetting := model2.TaskSetting{} err = util.FromJSONBytes(vBytes, &taskSetting) if err != nil { return nil, err @@ -70,7 +70,7 @@ func ParseAgentSettings(settings []agent.Setting)(*model.ParseAgentSettingsResul pipelines = append(pipelines, partPipelines...) toDeletePipelineNames = append(toDeletePipelineNames, partDeletePipelineNames...) } - return &model.ParseAgentSettingsResult{ + return &model2.ParseAgentSettingsResult{ ClusterConfigs: clusterCfgs, Pipelines: pipelines, ToDeletePipelineNames: toDeletePipelineNames, @@ -80,7 +80,7 @@ func ParseAgentSettings(settings []agent.Setting)(*model.ParseAgentSettingsResul // GetAgentSettings query agent setting by agent id and updated timestamp, // if there has any setting was updated, then return setting list includes settings not changed, // otherwise return empty setting list -func GetAgentSettings(agentID string, timestamp int64) ([]agent.Setting, error) { +func GetAgentSettings(agentID string, timestamp int64) ([]model.Setting, error) { query := util.MapStr{ "bool": util.MapStr{ "must": []util.MapStr{ @@ -116,13 +116,13 @@ func GetAgentSettings(agentID string, timestamp int64) ([]agent.Setting, error) }, } queryDsl := util.MapStr{ - "size": 500, + "size": 1000, "query": query, } q := orm.Query{ RawQuery: util.MustToJSONBytes(queryDsl), } - err, result := orm.Search(agent.Setting{}, &q) + err, result := orm.Search(model.Setting{}, &q) if err != nil { return nil, fmt.Errorf("search settings error: %w", err) } @@ -130,11 +130,11 @@ func GetAgentSettings(agentID string, timestamp int64) ([]agent.Setting, error) return nil, nil } var ( - settings []agent.Setting + settings []model.Setting hasUpdated bool ) for _, row := range result.Result { - setting := agent.Setting{} + setting := model.Setting{} buf, err := util.ToJSONBytes(row) if err != nil { return nil, err @@ -158,7 +158,7 @@ func getClusterConfigReferenceName(clusterID, nodeUUID string) string { return fmt.Sprintf("%s_%s", clusterID, nodeUUID) } -func TransformSettingsToConfig(setting *model.TaskSetting, clusterID, nodeUUID string) ([]util.MapStr, []string, error) { +func TransformSettingsToConfig(setting *model2.TaskSetting, clusterID, nodeUUID string) ([]util.MapStr, []string, error) { if setting == nil { return nil, nil, fmt.Errorf("empty setting") } @@ -279,6 +279,7 @@ func newClusterMetricPipeline(processorName string, clusterID string, nodeUUID s "name": getMetricPipelineName(clusterID, processorName), "auto_start": true, "keep_running": true, + "singleton": true, "retry_delay_in_ms": 10000, "processor": []util.MapStr{cfg}, } @@ -290,22 +291,22 @@ func getMetricPipelineName(clusterID, processorName string) string{ } -func LoadAgentsFromES(clusterID string) ([]agent.Instance, error) { +func LoadAgentsFromES(clusterID string) ([]model.Instance, error) { q := orm.Query{ Size: 1000, } if clusterID != "" { q.Conds = orm.And(orm.Eq("id", clusterID)) } - err, result := orm.Search(agent.Instance{}, &q) + err, result := orm.Search(model.Instance{}, &q) if err != nil { return nil, fmt.Errorf("query agent error: %w", err) } if len(result.Result) > 0 { - var agents = make([]agent.Instance, 0, len(result.Result)) + var agents = make([]model.Instance, 0, len(result.Result)) for _, row := range result.Result { - ag := agent.Instance{} + ag := model.Instance{} bytes := util.MustToJSONBytes(row) err = util.FromJSONBytes(bytes, &ag) if err != nil { @@ -417,7 +418,7 @@ func GetAgentIngestConfig() (string, *elastic.BasicAuth, error) { } if emptyIngestClusterEndpoint { cfg := elastic.GetConfig(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) - endpoint = cfg.Endpoint + endpoint = cfg.GetAnyEndpoint() } var ( @@ -441,101 +442,14 @@ func GetAgentIngestConfig() (string, *elastic.BasicAuth, error) { cfg := elastic.GetConfig(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) basicAuth = *cfg.BasicAuth } - tpl := `elasticsearch: - - name: default - enabled: true - endpoint: %s - discovery: - enabled: true - basic_auth: - username: %s - password: $[[keystore.ingest_cluster_password]] -metrics: - enabled: true - queue: metrics - network: - enabled: true - summary: true - details: true - memory: - metrics: - - swap - - memory - disk: - metrics: - - iops - - usage - cpu: - metrics: - - idle - - system - - user - - iowait - - load - instance: - enabled: true -pipeline: - - name: logs_indexing_merge - auto_start: true - keep_running: true - processor: - - indexing_merge: - index_name: ".infini_logs" - elasticsearch: "default" - input_queue: "logs" - idle_timeout_in_seconds: 10 - output_queue: - name: "logs_requests" - label: - tag: "logs" - worker_size: 1 - bulk_size_in_mb: 10 - - name: ingest_logs - auto_start: true - keep_running: true - processor: - - bulk_indexing: - bulk: - compress: true - batch_size_in_mb: 5 - batch_size_in_docs: 5000 - consumer: - fetch_max_messages: 100 - queues: - type: indexing_merge - tag: "logs" - when: - cluster_available: ["default"] - - name: metrics_indexing_merge - auto_start: true - keep_running: true - processor: - - indexing_merge: - elasticsearch: "default" - index_name: ".infini_metrics" - input_queue: "metrics" - output_queue: - name: "metrics_requests" - label: - tag: "metrics" - worker_size: 1 - bulk_size_in_mb: 5 - - name: ingest_metrics - auto_start: true - keep_running: true - processor: - - bulk_indexing: - bulk: - compress: true - batch_size_in_mb: 5 - batch_size_in_docs: 5000 - consumer: - fetch_max_messages: 100 - queues: - type: indexing_merge - tag: "metrics" - when: - cluster_available: ["default"]` + tpl := `configs.template: + - name: "ingest" + path: ./config/ingest_config.tpl + variable: + INGEST_CLUSTER_ID: "default_ingest_cluster" + INGEST_CLUSTER_ENDPOINT: ["%s"] + INGEST_CLUSTER_USERNAME: "%s" +` tpl = fmt.Sprintf(tpl, endpoint, basicAuth.Username) return tpl, &basicAuth, nil } \ No newline at end of file diff --git a/modules/agent/model/config.go b/modules/agent/model/config.go index f4bb4a2d..d731a3f8 100644 --- a/modules/agent/model/config.go +++ b/modules/agent/model/config.go @@ -5,20 +5,20 @@ package model type AgentConfig struct { - Enabled bool `config:"enabled"` - StateManager struct{ + Enabled bool `config:"enabled"` + StateManager struct { Enabled bool `config:"enabled"` } `config:"state_manager"` Setup *SetupConfig `config:"setup"` } type SetupConfig struct { - DownloadURL string `config:"download_url"` - Version string `config:"version"` - CACertFile string `config:"ca_cert"` - CAKeyFile string `config:"ca_key"` - ConsoleEndpoint string `config:"console_endpoint"` - IngestClusterEndpoint interface{} `config:"ingest_cluster_endpoint"` - IngestClusterCredentialID string `config:"ingest_cluster_credential_id"` - Port string `config:"port"` + DownloadURL string `config:"download_url"` + Version string `config:"version"` + CACertFile string `config:"ca_cert"` + CAKeyFile string `config:"ca_key"` + ConsoleEndpoint string `config:"console_endpoint"` + IngestClusterEndpoint interface{} `config:"ingest_cluster_endpoint"` + IngestClusterCredentialID string `config:"ingest_cluster_credential_id"` + Port string `config:"port"` } diff --git a/modules/agent/state/state.go b/modules/agent/state/state.go index b833f956..a4ff8342 100644 --- a/modules/agent/state/state.go +++ b/modules/agent/state/state.go @@ -12,8 +12,8 @@ import ( "gopkg.in/yaml.v2" "infini.sh/console/modules/agent/client" "infini.sh/console/modules/agent/common" - "infini.sh/console/modules/agent/model" - "infini.sh/framework/core/agent" + model2 "infini.sh/console/modules/agent/model" + "infini.sh/framework/core/model" "infini.sh/framework/core/host" "infini.sh/framework/core/kv" "infini.sh/framework/core/orm" @@ -44,9 +44,9 @@ func IsEnabled() bool { } type IStateManager interface { - GetAgent(ID string) (*agent.Instance, error) - UpdateAgent(inst *agent.Instance, syncToES bool) (*agent.Instance, error) - GetTaskAgent(clusterID string) (*agent.Instance, error) + GetAgent(ID string) (*model.Instance, error) + UpdateAgent(inst *model.Instance, syncToES bool) (*model.Instance, error) + GetTaskAgent(clusterID string) (*model.Instance, error) DeleteAgent(agentID string) error LoopState() Stop() @@ -87,7 +87,7 @@ func (sm *StateManager) checkAgentStatus() { for agentID := range onlineAgentIDs { if _, ok := sm.agentIds[agentID]; !ok { log.Infof("status of agent [%s] changed to online", agentID) - sm.agentIds[agentID] = model.StatusOnline + sm.agentIds[agentID] = model2.StatusOnline } } sm.agentMutex.Unlock() @@ -104,25 +104,25 @@ func (sm *StateManager) checkAgentStatus() { sm.syncSettings(agentID) sm.syncIngestSettings(agentID) if _, ok := onlineAgentIDs[agentID]; ok { - host.UpdateHostAgentStatus(agentID, model.StatusOnline) - if status == model.StatusOnline { + host.UpdateHostAgentStatus(agentID, model2.StatusOnline) + if status == model2.StatusOnline { return } // status change to online sm.agentMutex.Lock() - sm.agentIds[agentID] = model.StatusOnline + sm.agentIds[agentID] = model2.StatusOnline sm.agentMutex.Unlock() log.Infof("status of agent [%s] changed to online", agentID) return }else{ // already offline - if status == model.StatusOffline { + if status == model2.StatusOffline { return } } // status change to offline sm.agentMutex.Lock() - sm.agentIds[agentID] = model.StatusOffline + sm.agentIds[agentID] = model2.StatusOffline sm.agentMutex.Unlock() ag, err := sm.GetAgent(agentID) if err != nil { @@ -131,7 +131,7 @@ func (sm *StateManager) checkAgentStatus() { } return } - ag.Status = model.StatusOffline + ag.Status = model2.StatusOffline log.Infof("agent [%s] is offline", ag.Endpoint) _, err = sm.UpdateAgent(ag, true) if err != nil { @@ -139,13 +139,13 @@ func (sm *StateManager) checkAgentStatus() { return } //update host agent status - host.UpdateHostAgentStatus(ag.ID, model.StatusOffline) + host.UpdateHostAgentStatus(ag.ID, model2.StatusOffline) }(agentID) } } func (sm *StateManager) getLastSyncSettingsTimestamp(agentID string) int64{ - vbytes, err := kv.GetValue(model.KVSyncDynamicTaskSettings, []byte(agentID)) + vbytes, err := kv.GetValue(model2.KVSyncDynamicTaskSettings, []byte(agentID)) if err != nil { log.Error(err) } @@ -191,7 +191,7 @@ func (sm *StateManager) syncSettings(agentID string) { clusterCfg := util.MapStr{ "name": cfg.ID, "enabled": true, - "endpoint": cfg.Endpoint, + "endpoint": cfg.GetAnyEndpoint(), } if cfg.BasicAuth != nil && cfg.BasicAuth.Password != ""{ cid := cfg.ID @@ -223,16 +223,17 @@ func (sm *StateManager) syncSettings(agentID string) { log.Error("serialize config to yaml error: ", err) return } - err = agClient.SaveDynamicConfig(context.Background(), ag.GetEndpoint(), "dynamic_task", string(cfgBytes)) + //TODO + err = agClient.SaveDynamicConfig(context.Background(), ag.GetEndpoint(), "dynamic_task.yml", string(cfgBytes)) newTimestampStr := strconv.FormatInt(newTimestamp, 10) - err = kv.AddValue(model.KVSyncDynamicTaskSettings, []byte(agentID), []byte(newTimestampStr)) + err = kv.AddValue(model2.KVSyncDynamicTaskSettings, []byte(agentID), []byte(newTimestampStr)) if err != nil { log.Error(err) } } func (sm *StateManager) syncIngestSettings(agentID string) { - v, err := kv.GetValue(model.KVAgentIngestConfigChanged, []byte(agentID)) + v, err := kv.GetValue(model2.KVAgentIngestConfigChanged, []byte(agentID)) if err != nil { log.Error(err) } @@ -248,11 +249,11 @@ func (sm *StateManager) syncIngestSettings(agentID string) { } err = sm.agentClient.SaveIngestConfig(context.Background(), ag.GetEndpoint()) if err == nil { - kv.AddValue(model.KVAgentIngestConfigChanged,[]byte(agentID), []byte("0")) + kv.AddValue(model2.KVAgentIngestConfigChanged,[]byte(agentID), []byte("0")) } } -func (sm *StateManager) getAvailableAgent(clusterID string) (*agent.Instance, error) { +func (sm *StateManager) getAvailableAgent(clusterID string) (*model.Instance, error) { agents, err := common.LoadAgentsFromES(clusterID) if err != nil { return nil, err @@ -289,14 +290,14 @@ func (sm *StateManager) Stop() { <-sm.stopCompleteC } -func (sm *StateManager) GetAgent(ID string) (*agent.Instance, error) { +func (sm *StateManager) GetAgent(ID string) (*model.Instance, error) { buf, err := kv.GetValue(sm.KVKey, []byte(ID)) if err != nil { return nil, err } strTime, _ := jsonparser.GetString(buf, "timestamp") timestamp, _ := time.Parse(time.RFC3339, strTime) - inst := &agent.Instance{} + inst := &model.Instance{} inst.ID = ID if time.Since(timestamp) > sm.TTL { exists, err := orm.Get(inst) @@ -317,7 +318,7 @@ func (sm *StateManager) GetAgent(ID string) (*agent.Instance, error) { return inst, err } -func (sm *StateManager) UpdateAgent(inst *agent.Instance, syncToES bool) (*agent.Instance, error) { +func (sm *StateManager) UpdateAgent(inst *model.Instance, syncToES bool) (*model.Instance, error) { //inst.Timestamp = time.Now() err := kv.AddValue(sm.KVKey, []byte(inst.ID), util.MustToJSONBytes(inst)) if syncToES { @@ -332,7 +333,7 @@ func (sm *StateManager) UpdateAgent(inst *agent.Instance, syncToES bool) (*agent return inst, err } -func (sm *StateManager) GetTaskAgent(clusterID string) (*agent.Instance, error) { +func (sm *StateManager) GetTaskAgent(clusterID string) (*model.Instance, error) { return nil, nil } diff --git a/plugin/api/gateway/api.go b/plugin/api/gateway/api.go index 6ef82a9e..ad6d5b09 100644 --- a/plugin/api/gateway/api.go +++ b/plugin/api/gateway/api.go @@ -5,14 +5,7 @@ package gateway import ( - "crypto/tls" "infini.sh/framework/core/api" - "infini.sh/framework/core/api/rbac/enum" - "net" - "net/http" - "net/url" - log "github.com/cihub/seelog" - "time" ) type GatewayAPI struct { @@ -20,47 +13,48 @@ type GatewayAPI struct { } func InitAPI() { - gateway:=GatewayAPI{} - api.HandleAPIMethod(api.POST, "/gateway/instance/try_connect", gateway.RequireLogin(gateway.tryConnect)) - api.HandleAPIMethod(api.GET, "/gateway/instance/:instance_id", gateway.RequirePermission(gateway.getInstance, enum.PermissionGatewayInstanceRead)) - api.HandleAPIMethod(api.POST, "/gateway/instance", gateway.RequirePermission(gateway.createInstance, enum.PermissionGatewayInstanceWrite)) - api.HandleAPIMethod(api.PUT, "/gateway/instance/:instance_id", gateway.RequirePermission(gateway.updateInstance, enum.PermissionGatewayInstanceWrite)) - api.HandleAPIMethod(api.DELETE, "/gateway/instance/:instance_id", gateway.RequirePermission(gateway.deleteInstance, enum.PermissionGatewayInstanceWrite)) - api.HandleAPIMethod(api.GET, "/gateway/instance/_search", gateway.RequirePermission(gateway.searchInstance, enum.PermissionGatewayInstanceRead)) - api.HandleAPIMethod(api.POST, "/gateway/instance/status", gateway.RequirePermission(gateway.getInstanceStatus, enum.PermissionGatewayInstanceRead)) + //gateway:=GatewayAPI{} + //api.HandleAPIMethod(api.POST, "/gateway/instance/try_connect", gateway.RequireLogin(gateway.tryConnect)) + //api.HandleAPIMethod(api.GET, "/gateway/instance/:instance_id", gateway.RequirePermission(gateway.getInstance, enum.PermissionGatewayInstanceRead)) + //api.HandleAPIMethod(api.POST, "/gateway/instance", gateway.RequirePermission(gateway.createInstance, enum.PermissionGatewayInstanceWrite)) + //api.HandleAPIMethod(api.PUT, "/gateway/instance/:instance_id", gateway.RequirePermission(gateway.updateInstance, enum.PermissionGatewayInstanceWrite)) + //api.HandleAPIMethod(api.DELETE, "/gateway/instance/:instance_id", gateway.RequirePermission(gateway.deleteInstance, enum.PermissionGatewayInstanceWrite)) + //api.HandleAPIMethod(api.GET, "/gateway/instance/_search", gateway.RequirePermission(gateway.searchInstance, enum.PermissionGatewayInstanceRead)) + //api.HandleAPIMethod(api.POST, "/gateway/instance/status", gateway.RequirePermission(gateway.getInstanceStatus, enum.PermissionGatewayInstanceRead)) + // + //api.HandleAPIMethod(api.POST, "/gateway/instance/:instance_id/_proxy", gateway.RequirePermission(gateway.proxy, enum.PermissionGatewayInstanceRead)) - api.HandleAPIMethod(api.POST, "/gateway/instance/:instance_id/_proxy", gateway.RequirePermission(gateway.proxy, enum.PermissionGatewayInstanceRead)) - - api.HandleAPIMethod(api.GET, "/_platform/nodes", gateway.getExecutionNodes) - api.HandleAPIFunc("/ws_proxy", func(w http.ResponseWriter, req *http.Request) { - log.Debug(req.RequestURI) - endpoint := req.URL.Query().Get("endpoint") - path := req.URL.Query().Get("path") - var tlsConfig = &tls.Config{ - InsecureSkipVerify: true, - } - target, err := url.Parse(endpoint) - if err != nil { - log.Error(err) - return - } - newURL, err := url.Parse(path) - if err != nil { - log.Error(err) - return - } - req.URL.Path = newURL.Path - req.URL.RawPath = newURL.RawPath - req.URL.RawQuery = "" - req.RequestURI = req.URL.RequestURI() - req.Header.Set("HOST", target.Host) - req.Host = target.Host - wsProxy := NewSingleHostReverseProxy(target) - wsProxy.Dial = (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - }).Dial - wsProxy.TLSClientConfig = tlsConfig - wsProxy.ServeHTTP(w, req) - }) + //api.HandleAPIMethod(api.GET, "/_platform/nodes", gateway.getExecutionNodes) + // + //api.HandleAPIFunc("/ws_proxy", func(w http.ResponseWriter, req *http.Request) { + // log.Debug(req.RequestURI) + // endpoint := req.URL.Query().Get("endpoint") + // path := req.URL.Query().Get("path") + // var tlsConfig = &tls.Config{ + // InsecureSkipVerify: true, + // } + // target, err := url.Parse(endpoint) + // if err != nil { + // log.Error(err) + // return + // } + // newURL, err := url.Parse(path) + // if err != nil { + // log.Error(err) + // return + // } + // req.URL.Path = newURL.Path + // req.URL.RawPath = newURL.RawPath + // req.URL.RawQuery = "" + // req.RequestURI = req.URL.RequestURI() + // req.Header.Set("HOST", target.Host) + // req.Host = target.Host + // wsProxy := NewSingleHostReverseProxy(target) + // wsProxy.Dial = (&net.Dialer{ + // Timeout: 30 * time.Second, + // KeepAlive: 30 * time.Second, + // }).Dial + // wsProxy.TLSClientConfig = tlsConfig + // wsProxy.ServeHTTP(w, req) + //}) } \ No newline at end of file diff --git a/plugin/api/gateway/instance.go b/plugin/api/gateway/instance.go index e07ccca2..3ffe59df 100644 --- a/plugin/api/gateway/instance.go +++ b/plugin/api/gateway/instance.go @@ -1,495 +1,455 @@ -/* Copyright © INFINI Ltd. All rights reserved. - * web: https://infinilabs.com - * mail: hello#infini.ltd */ - +///* Copyright © INFINI Ltd. All rights reserved. +// * web: https://infinilabs.com +// * mail: hello#infini.ltd */ +// package gateway +// +//import ( +// "fmt" +// log "github.com/cihub/seelog" +// "github.com/segmentio/encoding/json" +// "infini.sh/console/model" +// httprouter "infini.sh/framework/core/api/router" +// elastic2 "infini.sh/framework/core/elastic" +// "infini.sh/framework/core/model" +// "infini.sh/framework/core/orm" +// "infini.sh/framework/core/proxy" +// "infini.sh/framework/core/task" +// "infini.sh/framework/core/util" +// "infini.sh/framework/modules/elastic" +// "net/http" +// "net/url" +// "strconv" +// "strings" +// "time" +//) +// -import ( - "fmt" - log "github.com/cihub/seelog" - "github.com/segmentio/encoding/json" - "infini.sh/console/model" - "infini.sh/framework/core/agent" - httprouter "infini.sh/framework/core/api/router" - elastic2 "infini.sh/framework/core/elastic" - "infini.sh/framework/core/orm" - "infini.sh/framework/core/proxy" - "infini.sh/framework/core/task" - "infini.sh/framework/core/util" - "infini.sh/framework/modules/elastic" - "net/http" - "net/url" - "strconv" - "strings" - "time" -) - -func (h *GatewayAPI) createInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - var obj = &model.Instance{} - err := h.DecodeJSON(req, obj) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - - res, err := h.doConnect(obj.Endpoint, obj.BasicAuth) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - obj.ID = res.ID - - exists, err := orm.Get(obj) - if err != nil && err != elastic.ErrNotFound { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - if exists { - h.WriteError(w, "gateway instance already registered", http.StatusInternalServerError) - return - } - err = orm.Create(nil, obj) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - - h.WriteJSON(w, util.MapStr{ - "_id": obj.ID, - "result": "created", - }, 200) - -} - -func (h *GatewayAPI) getInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - id := ps.MustGetParameter("instance_id") - - obj := model.Instance{} - obj.ID = id - - exists, err := orm.Get(&obj) - if !exists || err != nil { - h.WriteJSON(w, util.MapStr{ - "_id": id, - "found": false, - }, http.StatusNotFound) - return - } - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - - h.WriteJSON(w, util.MapStr{ - "found": true, - "_id": id, - "_source": obj, - }, 200) -} - -func (h *GatewayAPI) updateInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - id := ps.MustGetParameter("instance_id") - obj := model.Instance{} - - obj.ID = id - exists, err := orm.Get(&obj) - if !exists || err != nil { - h.WriteJSON(w, util.MapStr{ - "_id": id, - "result": "not_found", - }, http.StatusNotFound) - return - } - - id = obj.ID - create := obj.Created - obj = model.Instance{} - err = h.DecodeJSON(req, &obj) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - - //protect - obj.ID = id - obj.Created = create - err = orm.Update(nil, &obj) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - - h.WriteJSON(w, util.MapStr{ - "_id": obj.ID, - "result": "updated", - }, 200) -} - -func (h *GatewayAPI) deleteInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - id := ps.MustGetParameter("instance_id") - - obj := model.Instance{} - obj.ID = id - - exists, err := orm.Get(&obj) - if !exists || err != nil { - h.WriteJSON(w, util.MapStr{ - "_id": id, - "result": "not_found", - }, http.StatusNotFound) - return - } - - //check reference - query := util.MapStr{ - "size": 1, - "query": util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "term": util.MapStr{ - "metadata.labels.permit_nodes.id": util.MapStr{ - "value": id, - }, - }, - }, - { - "terms": util.MapStr{ - "metadata.type": []string{"cluster_migration", "cluster_comparison"}, - }, - }, - }, - "must_not": []util.MapStr{ - { - "terms": util.MapStr{ - "status": []string{task.StatusError, task.StatusComplete}, - }, - }, - }, - }, - }, - } - q := &orm.Query{ - RawQuery: util.MustToJSONBytes(query), - } - err, result := orm.Search(task.Task{}, q) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - if len(result.Result) > 0 { - var taskId interface{} - if m, ok := result.Result[0].(map[string]interface{}); ok { - taskId = m["id"] - } - h.WriteError(w, fmt.Sprintf("failed to delete gateway instance [%s] since it is used by task [%v]", id, taskId), http.StatusInternalServerError) - return - } - - err = orm.Delete(nil, &obj) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - - h.WriteJSON(w, util.MapStr{ - "_id": obj.ID, - "result": "deleted", - }, 200) -} - -func (h *GatewayAPI) searchInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - - var ( - keyword = h.GetParameterOrDefault(req, "keyword", "") - queryDSL = `{"query":{"bool":{"must":[%s]}}, "size": %d, "from": %d}` - strSize = h.GetParameterOrDefault(req, "size", "20") - strFrom = h.GetParameterOrDefault(req, "from", "0") - mustBuilder = &strings.Builder{} - ) - if keyword != "" { - mustBuilder.WriteString(fmt.Sprintf(`{"query_string":{"default_field":"*","query": "%s"}}`, keyword)) - } - size, _ := strconv.Atoi(strSize) - if size <= 0 { - size = 20 - } - from, _ := strconv.Atoi(strFrom) - if from < 0 { - from = 0 - } - - q := orm.Query{} - queryDSL = fmt.Sprintf(queryDSL, mustBuilder.String(), size, from) - q.RawQuery = []byte(queryDSL) - - err, res := orm.Search(&model.Instance{}, &q) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - h.Write(w, res.Raw) -} - -func (h *GatewayAPI) getInstanceStatus(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - var instanceIDs = []string{} - err := h.DecodeJSON(req, &instanceIDs) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - if len(instanceIDs) == 0 { - h.WriteJSON(w, util.MapStr{}, http.StatusOK) - return - } - q := orm.Query{} - queryDSL := util.MapStr{ - "query": util.MapStr{ - "terms": util.MapStr{ - "_id": instanceIDs, - }, - }, - } - q.RawQuery = util.MustToJSONBytes(queryDSL) - - err, res := orm.Search(&model.Instance{}, &q) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - result := util.MapStr{} - for _, item := range res.Result { - instance := util.MapStr(item.(map[string]interface{})) - if err != nil { - log.Error(err) - continue - } - endpoint, _ := instance.GetValue("endpoint") - username, _ := instance.GetValue("basic_auth.username") - if username == nil { - username = "" - } - password, _ := instance.GetValue("basic_auth.password") - if password == nil { - password = "" - } - gid, _ := instance.GetValue("id") - res, err := proxy.DoProxyRequest(&proxy.Request{ - Endpoint: endpoint.(string), - Method: http.MethodGet, - Path: "/stats", - BasicAuth: agent.BasicAuth{ - Username: username.(string), - Password: password.(string), - }, - }) - if err != nil { - log.Error(err) - result[gid.(string)] = util.MapStr{} - continue - } - var resMap = util.MapStr{} - err = util.FromJSONBytes(res.Body, &resMap) - if err != nil { - result[gid.(string)] = util.MapStr{} - log.Errorf("get stats of %v error: %v", endpoint, err) - continue - } - - result[gid.(string)] = resMap - } - h.WriteJSON(w, result, http.StatusOK) -} -func (h *GatewayAPI) proxy(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - var ( - method = h.Get(req, "method", "GET") - path = h.Get(req, "path", "") - ) - instanceID := ps.MustGetParameter("instance_id") - - obj := model.Instance{} - obj.ID = instanceID - - exists, err := orm.Get(&obj) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - if !exists { - h.WriteJSON(w, util.MapStr{ - "error": "gateway instance not found", - }, http.StatusNotFound) - return - } - res, err := proxy.DoProxyRequest(&proxy.Request{ - Method: method, - Endpoint: obj.Endpoint, - Path: path, - Body: req.Body, - BasicAuth: obj.BasicAuth, - ContentLength: int(req.ContentLength), - }) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - h.WriteHeader(w, res.StatusCode) - h.Write(w, res.Body) -} - -type GatewayConnectResponse struct { - ID string `json:"id"` - Name string `json:"name"` - Tagline string `json:"tagline"` - Version struct { - BuildDate string `json:"build_date"` - BuildHash string `json:"build_hash"` - EOLDate string `json:"eol_date"` - Number string `json:"number"` - } `json:"version"` - -} -func (h *GatewayAPI) doConnect(endpoint string, basicAuth agent.BasicAuth) (*GatewayConnectResponse, error) { - res, err := proxy.DoProxyRequest(&proxy.Request{ - Method: http.MethodGet, - Endpoint: endpoint, - Path: "/_framework/api/_info", - BasicAuth: basicAuth, - }) - if err != nil { - return nil, err - } - if res.StatusCode == http.StatusNotFound { - return nil, fmt.Errorf("unknow gateway version") - } - b := res.Body - gres := &GatewayConnectResponse{} - err = json.Unmarshal(b, gres) - return gres, err - -} - -func (h *GatewayAPI) tryConnect(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - var reqBody = struct { - Endpoint string `json:"endpoint"` - BasicAuth agent.BasicAuth - }{} - err := h.DecodeJSON(req, &reqBody) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - connectRes, err := h.doConnect(reqBody.Endpoint, reqBody.BasicAuth) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - h.WriteJSON(w, connectRes, http.StatusOK) -} - -func (h *GatewayAPI) getExecutionNodes(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ - var ( - keyword = h.GetParameterOrDefault(req, "keyword", "") - strSize = h.GetParameterOrDefault(req, "size", "10") - strFrom = h.GetParameterOrDefault(req, "from", "0") - ) - size, _ := strconv.Atoi(strSize) - if size <= 0 { - size = 10 - } - from, _ := strconv.Atoi(strFrom) - if from < 0 { - from = 0 - } - gatewayIndexName := orm.GetIndexName(model.Instance{}) - - query := util.MapStr{ - "size": size, - "from": from, - "sort": []util.MapStr{ - { - "created": util.MapStr{ - "order": "desc", - }, - }, - }, - } - if keyword != "" { - query["query"] = util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "prefix": util.MapStr{ - "name": util.MapStr{ - "value": keyword, - }, - }, - }, - }, - }, - } - } - q := orm.Query{ - IndexName: gatewayIndexName, - RawQuery: util.MustToJSONBytes(query), - } - err, result := orm.Search(nil, &q) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - searchRes := elastic2.SearchResponse{} - err = util.FromJSONBytes(result.Raw, &searchRes) - if err != nil||searchRes.ESError!=nil { - msg:=fmt.Sprintf("%v,%v",err,searchRes.ESError) - h.WriteError(w, msg, http.StatusInternalServerError) - return - } - var nodes = []util.MapStr{} - - for _, hit := range searchRes.Hits.Hits { - buf := util.MustToJSONBytes(hit.Source) - inst := model.Instance{} - err = util.FromJSONBytes(buf, &inst) - if err != nil { - log.Error(err) - continue - } - node := util.MapStr{ - "id": inst.ID, - "name": inst.Name, - "available": false, - "type": "gateway", - } - ul, err := url.Parse(inst.Endpoint) - if err != nil { - log.Error(err) - continue - } - node["host"] = ul.Host - err = inst.TryConnectWithTimeout(time.Second) - if err != nil { - log.Error(err) - }else{ - node["available"] = true - } - - nodes = append(nodes, node) - } - h.WriteJSON(w, nodes, http.StatusOK) -} +//func (h *GatewayAPI) getInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +// id := ps.MustGetParameter("instance_id") +// +// obj := model.Instance{} +// obj.ID = id +// +// exists, err := orm.Get(&obj) +// if !exists || err != nil { +// h.WriteJSON(w, util.MapStr{ +// "_id": id, +// "found": false, +// }, http.StatusNotFound) +// return +// } +// if err != nil { +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// log.Error(err) +// return +// } +// +// h.WriteJSON(w, util.MapStr{ +// "found": true, +// "_id": id, +// "_source": obj, +// }, 200) +//} +// +//func (h *GatewayAPI) updateInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +// id := ps.MustGetParameter("instance_id") +// obj := model.Instance{} +// +// obj.ID = id +// exists, err := orm.Get(&obj) +// if !exists || err != nil { +// h.WriteJSON(w, util.MapStr{ +// "_id": id, +// "result": "not_found", +// }, http.StatusNotFound) +// return +// } +// +// id = obj.ID +// create := obj.Created +// obj = model.Instance{} +// err = h.DecodeJSON(req, &obj) +// if err != nil { +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// log.Error(err) +// return +// } +// +// //protect +// obj.ID = id +// obj.Created = create +// err = orm.Update(nil, &obj) +// if err != nil { +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// log.Error(err) +// return +// } +// +// h.WriteJSON(w, util.MapStr{ +// "_id": obj.ID, +// "result": "updated", +// }, 200) +//} +// +//func (h *GatewayAPI) deleteInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +// id := ps.MustGetParameter("instance_id") +// +// obj := model.Instance{} +// obj.ID = id +// +// exists, err := orm.Get(&obj) +// if !exists || err != nil { +// h.WriteJSON(w, util.MapStr{ +// "_id": id, +// "result": "not_found", +// }, http.StatusNotFound) +// return +// } +// +// //check reference +// query := util.MapStr{ +// "size": 1, +// "query": util.MapStr{ +// "bool": util.MapStr{ +// "must": []util.MapStr{ +// { +// "term": util.MapStr{ +// "metadata.labels.permit_nodes.id": util.MapStr{ +// "value": id, +// }, +// }, +// }, +// { +// "terms": util.MapStr{ +// "metadata.type": []string{"cluster_migration", "cluster_comparison"}, +// }, +// }, +// }, +// "must_not": []util.MapStr{ +// { +// "terms": util.MapStr{ +// "status": []string{task.StatusError, task.StatusComplete}, +// }, +// }, +// }, +// }, +// }, +// } +// q := &orm.Query{ +// RawQuery: util.MustToJSONBytes(query), +// } +// err, result := orm.Search(task.Task{}, q) +// if err != nil { +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// log.Error(err) +// return +// } +// if len(result.Result) > 0 { +// var taskId interface{} +// if m, ok := result.Result[0].(map[string]interface{}); ok { +// taskId = m["id"] +// } +// h.WriteError(w, fmt.Sprintf("failed to delete gateway instance [%s] since it is used by task [%v]", id, taskId), http.StatusInternalServerError) +// return +// } +// +// err = orm.Delete(nil, &obj) +// if err != nil { +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// log.Error(err) +// return +// } +// +// h.WriteJSON(w, util.MapStr{ +// "_id": obj.ID, +// "result": "deleted", +// }, 200) +//} +// +//func (h *GatewayAPI) searchInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +// +// var ( +// keyword = h.GetParameterOrDefault(req, "keyword", "") +// queryDSL = `{"query":{"bool":{"must":[%s]}}, "size": %d, "from": %d}` +// strSize = h.GetParameterOrDefault(req, "size", "20") +// strFrom = h.GetParameterOrDefault(req, "from", "0") +// mustBuilder = &strings.Builder{} +// ) +// if keyword != "" { +// mustBuilder.WriteString(fmt.Sprintf(`{"query_string":{"default_field":"*","query": "%s"}}`, keyword)) +// } +// size, _ := strconv.Atoi(strSize) +// if size <= 0 { +// size = 20 +// } +// from, _ := strconv.Atoi(strFrom) +// if from < 0 { +// from = 0 +// } +// +// q := orm.Query{} +// queryDSL = fmt.Sprintf(queryDSL, mustBuilder.String(), size, from) +// q.RawQuery = []byte(queryDSL) +// +// err, res := orm.Search(&model.Instance{}, &q) +// if err != nil { +// log.Error(err) +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// return +// } +// h.Write(w, res.Raw) +//} +// +//func (h *GatewayAPI) getInstanceStatus(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +// var instanceIDs = []string{} +// err := h.DecodeJSON(req, &instanceIDs) +// if err != nil { +// log.Error(err) +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// return +// } +// if len(instanceIDs) == 0 { +// h.WriteJSON(w, util.MapStr{}, http.StatusOK) +// return +// } +// q := orm.Query{} +// queryDSL := util.MapStr{ +// "query": util.MapStr{ +// "terms": util.MapStr{ +// "_id": instanceIDs, +// }, +// }, +// } +// q.RawQuery = util.MustToJSONBytes(queryDSL) +// +// err, res := orm.Search(&model.Instance{}, &q) +// if err != nil { +// log.Error(err) +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// return +// } +// result := util.MapStr{} +// for _, item := range res.Result { +// instance := util.MapStr(item.(map[string]interface{})) +// if err != nil { +// log.Error(err) +// continue +// } +// endpoint, _ := instance.GetValue("endpoint") +// username, _ := instance.GetValue("basic_auth.username") +// if username == nil { +// username = "" +// } +// password, _ := instance.GetValue("basic_auth.password") +// if password == nil { +// password = "" +// } +// gid, _ := instance.GetValue("id") +// res, err := proxy.DoProxyRequest(&proxy.Request{ +// Endpoint: endpoint.(string), +// Method: http.MethodGet, +// Path: "/stats", +// BasicAuth: instance.BasicAuth{ +// Username: username.(string), +// Password: password.(string), +// }, +// }) +// if err != nil { +// log.Error(err) +// result[gid.(string)] = util.MapStr{} +// continue +// } +// var resMap = util.MapStr{} +// err = util.FromJSONBytes(res.Body, &resMap) +// if err != nil { +// result[gid.(string)] = util.MapStr{} +// log.Errorf("get stats of %v error: %v", endpoint, err) +// continue +// } +// +// result[gid.(string)] = resMap +// } +// h.WriteJSON(w, result, http.StatusOK) +//} +//func (h *GatewayAPI) proxy(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +// var ( +// method = h.Get(req, "method", "GET") +// path = h.Get(req, "path", "") +// ) +// instanceID := ps.MustGetParameter("instance_id") +// +// obj := model.Instance{} +// obj.ID = instanceID +// +// exists, err := orm.Get(&obj) +// if err != nil { +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// log.Error(err) +// return +// } +// if !exists { +// h.WriteJSON(w, util.MapStr{ +// "error": "gateway instance not found", +// }, http.StatusNotFound) +// return +// } +// res, err := proxy.DoProxyRequest(&proxy.Request{ +// Method: method, +// Endpoint: obj.Endpoint, +// Path: path, +// Body: req.Body, +// BasicAuth: obj.BasicAuth, +// ContentLength: int(req.ContentLength), +// }) +// if err != nil { +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// log.Error(err) +// return +// } +// h.WriteHeader(w, res.StatusCode) +// h.Write(w, res.Body) +//} +// +//type GatewayConnectResponse struct { +// ID string `json:"id"` +// Name string `json:"name"` +// Tagline string `json:"tagline"` +// Version struct { +// BuildDate string `json:"build_date"` +// BuildHash string `json:"build_hash"` +// EOLDate string `json:"eol_date"` +// Number string `json:"number"` +// } `json:"version"` +// +//} +//func (h *GatewayAPI) doConnect(endpoint string, basicAuth model.BasicAuth) (*GatewayConnectResponse, error) { +// res, err := proxy.DoProxyRequest(&proxy.Request{ +// Method: http.MethodGet, +// Endpoint: endpoint, +// Path: "/_info", +// BasicAuth: basicAuth, +// }) +// if err != nil { +// return nil, err +// } +// if res.StatusCode == http.StatusNotFound { +// return nil, fmt.Errorf("unknow gateway version") +// } +// b := res.Body +// gres := &GatewayConnectResponse{} +// err = json.Unmarshal(b, gres) +// return gres, err +// +//} +// +//func (h *GatewayAPI) tryConnect(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +// var reqBody = struct { +// Endpoint string `json:"endpoint"` +// BasicAuth model.BasicAuth +// }{} +// err := h.DecodeJSON(req, &reqBody) +// if err != nil { +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// return +// } +// connectRes, err := h.doConnect(reqBody.Endpoint, reqBody.BasicAuth) +// if err != nil { +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// return +// } +// h.WriteJSON(w, connectRes, http.StatusOK) +//} +// +//func (h *GatewayAPI) getExecutionNodes(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ +// var ( +// keyword = h.GetParameterOrDefault(req, "keyword", "") +// strSize = h.GetParameterOrDefault(req, "size", "10") +// strFrom = h.GetParameterOrDefault(req, "from", "0") +// ) +// size, _ := strconv.Atoi(strSize) +// if size <= 0 { +// size = 10 +// } +// from, _ := strconv.Atoi(strFrom) +// if from < 0 { +// from = 0 +// } +// gatewayIndexName := orm.GetIndexName(model.Instance{}) +// +// query := util.MapStr{ +// "size": size, +// "from": from, +// "sort": []util.MapStr{ +// { +// "created": util.MapStr{ +// "order": "desc", +// }, +// }, +// }, +// } +// if keyword != "" { +// query["query"] = util.MapStr{ +// "bool": util.MapStr{ +// "must": []util.MapStr{ +// { +// "prefix": util.MapStr{ +// "name": util.MapStr{ +// "value": keyword, +// }, +// }, +// }, +// }, +// }, +// } +// } +// q := orm.Query{ +// IndexName: gatewayIndexName, +// RawQuery: util.MustToJSONBytes(query), +// } +// err, result := orm.Search(nil, &q) +// if err != nil { +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// return +// } +// +// searchRes := elastic2.SearchResponse{} +// err = util.FromJSONBytes(result.Raw, &searchRes) +// if err != nil||searchRes.ESError!=nil { +// msg:=fmt.Sprintf("%v,%v",err,searchRes.ESError) +// h.WriteError(w, msg, http.StatusInternalServerError) +// return +// } +// var nodes = []util.MapStr{} +// +// for _, hit := range searchRes.Hits.Hits { +// buf := util.MustToJSONBytes(hit.Source) +// inst := model.Instance{} +// err = util.FromJSONBytes(buf, &inst) +// if err != nil { +// log.Error(err) +// continue +// } +// node := util.MapStr{ +// "id": inst.ID, +// "name": inst.Name, +// "available": false, +// "type": "gateway", +// } +// ul, err := url.Parse(inst.Endpoint) +// if err != nil { +// log.Error(err) +// continue +// } +// node["host"] = ul.Host +// err = inst.TryConnectWithTimeout(time.Second) +// if err != nil { +// log.Error(err) +// }else{ +// node["available"] = true +// } +// +// nodes = append(nodes, node) +// } +// h.WriteJSON(w, nodes, http.StatusOK) +//} diff --git a/plugin/api/gateway/websocket_proxy.go b/plugin/api/gateway/websocket_proxy.go deleted file mode 100644 index 99fe929e..00000000 --- a/plugin/api/gateway/websocket_proxy.go +++ /dev/null @@ -1,186 +0,0 @@ -/* Copyright © INFINI Ltd. All rights reserved. - * Web: https://infinilabs.com - * Email: hello#infini.ltd */ - -package gateway - -import ( - "crypto/tls" - "io" - log "github.com/cihub/seelog" - "net" - "net/http" - "net/url" - "strings" -) - -// ReverseProxy is a WebSocket reverse proxy. It will not work with a regular -// HTTP request, so it is the caller's responsiblity to ensure the incoming -// request is a WebSocket request. -type ReverseProxy struct { - // Director must be a function which modifies - // the request into a new request to be sent - // using Transport. Its response is then copied - // back to the original client unmodified. - Director func(*http.Request) - - // Dial specifies the dial function for dialing the proxied - // server over tcp. - // If Dial is nil, net.Dial is used. - Dial func(network, addr string) (net.Conn, error) - - // TLSClientConfig specifies the TLS configuration to use for 'wss'. - // If nil, the default configuration is used. - TLSClientConfig *tls.Config - - // ErrorLog specifies an optional logger for errors - // that occur when attempting to proxy the request. - // If nil, logging goes to os.Stderr via the log package's - // standard logger. - ErrorLog *log.LoggerInterface -} - -// stolen from net/http/httputil. singleJoiningSlash ensures that the route -// '/a/' joined with '/b' becomes '/a/b'. -func singleJoiningSlash(a, b string) string { - aslash := strings.HasSuffix(a, "/") - bslash := strings.HasPrefix(b, "/") - switch { - case aslash && bslash: - return a + b[1:] - case !aslash && !bslash: - return a + "/" + b - } - return a + b -} - -// NewSingleHostReverseProxy returns a new websocket ReverseProxy. The path -// rewrites follow the same rules as the httputil.ReverseProxy. If the target -// url has the path '/foo' and the incoming request '/bar', the request path -// will be updated to '/foo/bar' before forwarding. -// Scheme should specify if 'ws' or 'wss' should be used. -func NewSingleHostReverseProxy(target *url.URL) *ReverseProxy { - targetQuery := target.RawQuery - director := func(req *http.Request) { - req.URL.Scheme = target.Scheme - req.URL.Host = target.Host - req.URL.Path = singleJoiningSlash(target.Path, req.URL.Path) - if targetQuery == "" || req.URL.RawQuery == "" { - req.URL.RawQuery = targetQuery + req.URL.RawQuery - } else { - req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery - } - } - return &ReverseProxy{Director: director} -} - -// Function to implement the http.Handler interface. -func (p *ReverseProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { - logFunc := log.Errorf - - if !IsWebSocketRequest(r) { - http.Error(w, "Cannot handle non-WebSocket requests", 500) - logFunc("Received a request that was not a WebSocket request") - return - } - - outreq := new(http.Request) - // shallow copying - *outreq = *r - p.Director(outreq) - host := outreq.URL.Host - - if clientIP, _, err := net.SplitHostPort(r.RemoteAddr); err == nil { - // If we aren't the first proxy retain prior - // X-Forwarded-For information as a comma+space - // separated list and fold multiple headers into one. - if prior, ok := outreq.Header["X-Forwarded-For"]; ok { - clientIP = strings.Join(prior, ", ") + ", " + clientIP - } - outreq.Header.Set("X-Forwarded-For", clientIP) - } - - dial := p.Dial - if dial == nil { - dial = net.Dial - } - - // if host does not specify a port, use the default http port - if !strings.Contains(host, ":") { - if outreq.URL.Scheme == "wss" { - host = host + ":443" - } else { - host = host + ":80" - } - } - - if outreq.URL.Scheme == "wss" { - var tlsConfig *tls.Config - if p.TLSClientConfig == nil { - tlsConfig = &tls.Config{} - } else { - tlsConfig = p.TLSClientConfig - } - dial = func(network, address string) (net.Conn, error) { - return tls.Dial("tcp", host, tlsConfig) - } - } - - d, err := dial("tcp", host) - if err != nil { - http.Error(w, "Error forwarding request.", 500) - logFunc("Error dialing websocket backend %s: %v", outreq.URL, err) - return - } - // All request generated by the http package implement this interface. - hj, ok := w.(http.Hijacker) - if !ok { - http.Error(w, "Not a hijacker?", 500) - return - } - // Hijack() tells the http package not to do anything else with the connection. - // After, it bcomes this functions job to manage it. `nc` is of type *net.Conn. - nc, _, err := hj.Hijack() - if err != nil { - logFunc("Hijack error: %v", err) - return - } - defer nc.Close() // must close the underlying net connection after hijacking - defer d.Close() - - // write the modified incoming request to the dialed connection - err = outreq.Write(d) - if err != nil { - logFunc("Error copying request to target: %v", err) - return - } - errc := make(chan error, 2) - cp := func(dst io.Writer, src io.Reader) { - _, err := io.Copy(dst, src) - errc <- err - } - go cp(d, nc) - go cp(nc, d) - <-errc -} - -// IsWebSocketRequest returns a boolean indicating whether the request has the -// headers of a WebSocket handshake request. -func IsWebSocketRequest(r *http.Request) bool { - contains := func(key, val string) bool { - vv := strings.Split(r.Header.Get(key), ",") - for _, v := range vv { - if val == strings.ToLower(strings.TrimSpace(v)) { - return true - } - } - return false - } - if !contains("Connection", "upgrade") { - return false - } - if !contains("Upgrade", "websocket") { - return false - } - return true -} diff --git a/plugin/setup/setup.go b/plugin/setup/setup.go index 7cd0f6e0..b746a653 100644 --- a/plugin/setup/setup.go +++ b/plugin/setup/setup.go @@ -495,10 +495,6 @@ func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps http } } - if err != nil { - panic(err) - } - //处理索引 elastic2.InitSchema() //init security @@ -573,7 +569,7 @@ func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps http _, err = util.FilePutContent(file, fmt.Sprintf("configs.template:\n - name: \"system\"\n path: ./config/system_config.tpl\n variable:\n "+ "CLUSTER_ID: %v\n CLUSTER_ENDPINT: \"%v\"\n "+ "CLUSTER_USER: \"%v\"\n CLUSTER_VER: \"%v\"\n CLUSTER_DISTRIBUTION: \"%v\"\n INDEX_PREFIX: \"%v\"", - GlobalSystemElasticsearchID, cfg.Endpoint, cfg.BasicAuth.Username, cfg.Version, cfg.Distribution, cfg1.IndexPrefix)) + GlobalSystemElasticsearchID, cfg.GetAnyEndpoint(), cfg.BasicAuth.Username, cfg.Version, cfg.Distribution, cfg1.IndexPrefix)) if err != nil { panic(err) } diff --git a/plugin/task_manager/common_api.go b/plugin/task_manager/common_api.go index 08241dda..087f6ce3 100644 --- a/plugin/task_manager/common_api.go +++ b/plugin/task_manager/common_api.go @@ -3,6 +3,7 @@ package task_manager import ( "errors" "fmt" + model2 "infini.sh/console/model" "net/http" "strconv" "strings" @@ -10,7 +11,6 @@ import ( log "github.com/cihub/seelog" - "infini.sh/console/model" migration_util "infini.sh/console/plugin/task_manager/util" httprouter "infini.sh/framework/core/api/router" @@ -560,7 +560,7 @@ func (h *APIHandler) getChildPipelineInfosFromGateway(pipelineTaskIDs map[string var err error for instID, taskIDs := range pipelineTaskIDs { - inst := &model.Instance{} + inst := &model2.TaskWorker{} inst.ID = instID _, err = orm.Get(inst) if err != nil { diff --git a/plugin/task_manager/model/scheduler.go b/plugin/task_manager/model/scheduler.go index d1526f5f..cd595ae2 100644 --- a/plugin/task_manager/model/scheduler.go +++ b/plugin/task_manager/model/scheduler.go @@ -2,13 +2,12 @@ package model import ( "errors" - "infini.sh/console/model" ) type Scheduler interface { - GetPreferenceInstance(config ExecutionConfig) (instance *model.Instance, err error) - GetInstance(instanceID string) (instance *model.Instance, err error) + GetPreferenceInstance(config ExecutionConfig) (instance *model.TaskWorker, err error) + GetInstance(instanceID string) (instance *model.TaskWorker, err error) IncrInstanceJobs(instanceID string) DecrInstanceJobs(instanceID string) RefreshInstanceJobsFromES() error diff --git a/plugin/task_manager/pipeline_task/pipeline_task.go b/plugin/task_manager/pipeline_task/pipeline_task.go index 0c8e0133..6b41326e 100644 --- a/plugin/task_manager/pipeline_task/pipeline_task.go +++ b/plugin/task_manager/pipeline_task/pipeline_task.go @@ -3,13 +3,13 @@ package pipeline_task import ( "errors" "fmt" + "infini.sh/console/model" "strconv" "strings" "time" log "github.com/cihub/seelog" - "infini.sh/console/model" migration_model "infini.sh/console/plugin/task_manager/model" migration_util "infini.sh/console/plugin/task_manager/util" @@ -169,7 +169,7 @@ func (p *processor) handlePendingStopPipelineTask(taskItem *task.Task) error { return nil } -func (p *processor) cleanGatewayPipeline(taskItem *task.Task) (instance *model.Instance, err error) { +func (p *processor) cleanGatewayPipeline(taskItem *task.Task) (instance *model.TaskWorker, err error) { instance, err = p.getPipelineExecutionInstance(taskItem) if err != nil { log.Errorf("failed to get execution instance for task [%s], err: %v", taskItem.ID, err) @@ -184,7 +184,7 @@ func (p *processor) cleanGatewayPipeline(taskItem *task.Task) (instance *model.I return instance, nil } -func (p *processor) getPipelineExecutionInstance(taskItem *task.Task) (*model.Instance, error) { +func (p *processor) getPipelineExecutionInstance(taskItem *task.Task) (*model.TaskWorker, error) { instanceID, _ := util.ExtractString(taskItem.Metadata.Labels["execution_instance_id"]) instance, err := p.scheduler.GetInstance(instanceID) if err != nil { diff --git a/plugin/task_manager/scheduler/scheduler.go b/plugin/task_manager/scheduler/scheduler.go index 1d7413c4..66ebea41 100644 --- a/plugin/task_manager/scheduler/scheduler.go +++ b/plugin/task_manager/scheduler/scheduler.go @@ -53,7 +53,7 @@ func NewScheduler(elasticsearch, indexName string, checkInstanceAvailable bool, return scheduler, nil } -func (p *scheduler) GetPreferenceInstance(config migration_model.ExecutionConfig) (*model.Instance, error) { +func (p *scheduler) GetPreferenceInstance(config migration_model.ExecutionConfig) (*model.TaskWorker, error) { var ( err error minID string @@ -64,7 +64,7 @@ func (p *scheduler) GetPreferenceInstance(config migration_model.ExecutionConfig instanceTotal := p.getInstanceState(node.ID).Total if instanceTotal < minTotal { if p.CheckInstanceAvailable { - tempInst := model.Instance{} + tempInst := model.TaskWorker{} tempInst.ID = node.ID _, err = orm.Get(&tempInst) if err != nil { @@ -95,11 +95,11 @@ func (p *scheduler) GetPreferenceInstance(config migration_model.ExecutionConfig return instance, nil } -func (p *scheduler) GetInstance(instanceID string) (*model.Instance, error) { +func (p *scheduler) GetInstance(instanceID string) (*model.TaskWorker, error) { if instanceID == "" { return nil, errors.New("invalid instanceID") } - instance := model.Instance{} + instance := model.TaskWorker{} instance.ID = instanceID _, err := orm.Get(&instance) @@ -114,7 +114,7 @@ func (p *scheduler) GetInstance(instanceID string) (*model.Instance, error) { return &instance, nil } -func (p *scheduler) initializeInstance(instance *model.Instance) error { +func (p *scheduler) initializeInstance(instance *model.TaskWorker) error { lastInitializedAt := p.getLastInitializedAt(instance.ID) if time.Now().Sub(lastInitializedAt) < initializeInterval { return nil @@ -162,7 +162,7 @@ func (p *scheduler) initializeInstance(instance *model.Instance) error { // user could change the following configurations manually: // - input_queue (metrics.logging_queue) // - elasticsearch (elasticsearch.name) -func (p *scheduler) createPipelineLoggingMerge(instance *model.Instance) error { +func (p *scheduler) createPipelineLoggingMerge(instance *model.TaskWorker) error { cfg := &migration_model.PipelineTaskConfig{ Name: "pipeline_logging_merge", AutoStart: true, @@ -194,7 +194,7 @@ func (p *scheduler) createPipelineLoggingMerge(instance *model.Instance) error { return nil } -func (p *scheduler) createIngestPipelineLogging(instance *model.Instance) error { +func (p *scheduler) createIngestPipelineLogging(instance *model.TaskWorker) error { cfg := &migration_model.PipelineTaskConfig{ Name: "ingest_pipeline_logging", AutoStart: true, diff --git a/service/alerting/env.go b/service/alerting/env.go index b1f3c340..876ad91b 100644 --- a/service/alerting/env.go +++ b/service/alerting/env.go @@ -6,8 +6,8 @@ package alerting import ( "fmt" - "infini.sh/framework/core/config" config2 "infini.sh/console/config" + "infini.sh/framework/core/config" "infini.sh/framework/core/env" "infini.sh/framework/core/global" "infini.sh/framework/core/kv" @@ -53,6 +53,6 @@ func GetInnerConsoleEndpoint() (string, error){ if !ok { return "", fmt.Errorf("web config not exists") } - endpoint := fmt.Sprintf("%s://%s", appConfig.GetSchema(), appConfig.Network.GetPublishAddr()) + endpoint := appConfig.GetEndpoint() return endpoint, nil } From a315c4330d9c581bdedb211afff8d6cf2afd2b8b Mon Sep 17 00:00:00 2001 From: medcl Date: Mon, 16 Oct 2023 17:51:45 +0800 Subject: [PATCH 04/36] fix build --- config/install_agent.tpl | 26 +- config_repo/configs/cluster_xx_node_xx.yml | 17 + config_repo/configs/ingest_config.yml | 14 + config_repo/settings.yml | 31 + config_repo/templates/ingest_config.tpl | 97 +++ config_repo/templates/task_config.tpl | 87 ++ console.yml | 18 +- main.go | 9 +- model/email_server.go | 4 +- modules/agent/api/elasticsearch.go | 353 +++++++++ modules/agent/api/host.go | 436 +++++----- modules/agent/api/init.go | 47 +- modules/agent/api/instance.go | 1 + modules/agent/api/log.go | 124 --- modules/agent/api/tod.go | 881 +++++++++++++++++++++ modules/agent/client/client.go | 279 ------- modules/agent/client/executor.go | 100 --- modules/agent/common/config.go | 36 +- modules/agent/common/helper.go | 6 +- modules/agent/common/helper_test.go | 41 - modules/agent/model/const.go | 15 - modules/agent/state/state.go | 352 -------- plugin/api/email/common/auth.go | 6 +- plugin/api/license/api.go | 2 +- plugin/setup/setup.go | 7 +- 25 files changed, 1780 insertions(+), 1209 deletions(-) create mode 100644 config_repo/configs/cluster_xx_node_xx.yml create mode 100644 config_repo/configs/ingest_config.yml create mode 100644 config_repo/settings.yml create mode 100644 config_repo/templates/ingest_config.tpl create mode 100644 config_repo/templates/task_config.tpl create mode 100644 modules/agent/api/elasticsearch.go delete mode 100644 modules/agent/api/log.go create mode 100644 modules/agent/api/tod.go delete mode 100644 modules/agent/client/client.go delete mode 100644 modules/agent/client/executor.go delete mode 100644 modules/agent/common/helper_test.go delete mode 100644 modules/agent/model/const.go delete mode 100644 modules/agent/state/state.go diff --git a/config/install_agent.tpl b/config/install_agent.tpl index e764d685..1b0881d5 100644 --- a/config/install_agent.tpl +++ b/config/install_agent.tpl @@ -3,7 +3,7 @@ set -eo pipefail function print_usage() { - echo "Usage: curl -sSL http://get.infini.sh/agent.html | sudo bash -s -- [-u url_for_download_program] [-v version_for_program ] [-t taget_install_dir] [-p prot_for_program]" + echo "Usage: curl -sSL http://get.infini.cloud/ | sudo bash -s -- [-u url_for_download_program] [-v version_for_program ] [-t target_install_dir] [-p port_for_program]" echo "Options:" echo " -u, --url Download url of the program to install which default is http://localhost" echo " -v, --version Version of the program to install which default is latest from " @@ -226,6 +226,15 @@ resource_limit.memory.max_in_bytes: 533708800 stats: include_storage_stats_in_api: false +elastic: + skip_init_metadata_on_start: true + health_check: + enabled: true + interval: 60s + availability_check: + enabled: false + interval: 60s + disk_queue: max_msg_size: 20485760 max_bytes_per_file: 20485760 @@ -254,6 +263,21 @@ badger: value_log_max_entries: 1000000 value_log_file_size: 104857600 +configs: + #for managed client's setting + managed: true # managed by remote servers + panic_on_config_error: false #ignore config error + interval: "1s" + servers: # config servers + - "http://localhost:9000" + max_backup_files: 5 + tls: #for mTLS connection with config servers + enabled: true + cert_file: "config/client.crt" + key_file: "config/client.key" + ca_file: "config/ca.crt" + skip_insecure_verify: false + node: major_ip_pattern: ".*" EOF diff --git a/config_repo/configs/cluster_xx_node_xx.yml b/config_repo/configs/cluster_xx_node_xx.yml new file mode 100644 index 00000000..d67e7cc6 --- /dev/null +++ b/config_repo/configs/cluster_xx_node_xx.yml @@ -0,0 +1,17 @@ +configs.template: + - name: "cluster_a_node_b_task_config" + path: ./config/task_config.tpl + variable: + CLUSTER_ID: infini_default_ingest_cluster + CLUSTER_ENDPOINT: ["https://localhost:9200"] + CLUSTER_USERNAME: "admin" + CLUSTER_VER: "1.6.0" + CLUSTER_DISTRIBUTION: "easysearch" + INDEX_PREFIX: ".infini_" + CLUSTER_LEVEL_TASKS_ENABLED: false + NODE_LEVEL_TASKS_ENABLED: true + NODE_LOGS_PATH: "/opt/easysearch/logs/" + + +#MANAGED_CONFIG_VERSION: 19 +#MANAGED: true \ No newline at end of file diff --git a/config_repo/configs/ingest_config.yml b/config_repo/configs/ingest_config.yml new file mode 100644 index 00000000..bb2440e9 --- /dev/null +++ b/config_repo/configs/ingest_config.yml @@ -0,0 +1,14 @@ +configs.template: + - name: "default_ingest_config" + path: ./config/ingest_config.tpl + variable: + INGEST_CLUSTER_ID: infini_default_ingest_cluster + INGEST_CLUSTER_ENDPOINT: [ "https://localhost:9200" ] + INGEST_CLUSTER_USERNAME: "admin" + CLUSTER_VER: "1.6.0" + CLUSTER_DISTRIBUTION: "easysearch" + INDEX_PREFIX: ".infini_" + + +#MANAGED_CONFIG_VERSION: 2 +#MANAGED: true \ No newline at end of file diff --git a/config_repo/settings.yml b/config_repo/settings.yml new file mode 100644 index 00000000..41d42024 --- /dev/null +++ b/config_repo/settings.yml @@ -0,0 +1,31 @@ +configs: #define configs group + general_ingest_template: #group name + files: + - ./templates/ingest_config.tpl + - ./templates/task_config.tpl + - ./configs/ingest_config.yml + - ./configs/cluster_xx_node_xx.yml +instances: #define which config instance should fetch + infini_default_system_cluster: #instance group + plugins: + - ingest + instances: + - ck0mkk805f5virpsejp0 + - ckjrpdg05f5lrfp8qlng + configs: + - general_ingest_template + secrets: + - ingest_cluster_password + +secrets: + ingest_cluster_password: #group name + keystore: + ingest_cluster_password: + type: plaintext + value: "d7cc48e69a41dac719fb" + infini_default_ingest_cluster_password: + type: plaintext + value: "d7cc48e69a41dac719fb" +# infini_default_ingest_cluster_password: +# type: credential +# value: "ckghspo05f5q7pr20ct0" #credential_id \ No newline at end of file diff --git a/config_repo/templates/ingest_config.tpl b/config_repo/templates/ingest_config.tpl new file mode 100644 index 00000000..258dfe06 --- /dev/null +++ b/config_repo/templates/ingest_config.tpl @@ -0,0 +1,97 @@ +elasticsearch: + - name: $[[INGEST_CLUSTER_ID]] + enabled: true + endpoints: $[[INGEST_CLUSTER_ENDPOINT]] + discovery: + enabled: false + basic_auth: + username: $[[INGEST_CLUSTER_USERNAME]] + password: $[[keystore.ingest_cluster_password]] + +metrics: + enabled: true + queue: metrics + network: + enabled: true + summary: true + details: true + memory: + metrics: + - swap + - memory + disk: + metrics: + - iops + - usage + cpu: + metrics: + - idle + - system + - user + - iowait + - load + instance: + enabled: true + +elastic: + availability_check: + enabled: false + +pipeline: + - name: merge_logs + auto_start: true + keep_running: true + processor: + - indexing_merge: + elasticsearch: "$[[INGEST_CLUSTER_ID]]" + index_name: ".infini_logs" + type_name: "_doc" + input_queue: "logs" + idle_timeout_in_seconds: 10 + output_queue: + name: "merged_requests" + worker_size: 1 + bulk_size_in_mb: 5 + - name: merge_metrics + auto_start: true + keep_running: true + processor: + - indexing_merge: + elasticsearch: "$[[INGEST_CLUSTER_ID]]" + index_name: ".infini_metrics" + type_name: "_doc" + input_queue: "metrics" + output_queue: + name: "merged_requests" + worker_size: 1 + bulk_size_in_mb: 5 + - name: ingest_merged_requests + enabled: true + auto_start: true + keep_running: true + processor: + - bulk_indexing: + max_worker_size: 1 + verbose_bulk_result: false + bulk: + batch_size_in_mb: 5 + batch_size_in_docs: 5000 + max_retry_times: 0 + invalid_queue: "" + response_handle: + include_index_stats: false + include_action_stats: false + output_bulk_stats: false + include_error_details: true + save_error_results: true + save_success_results: false + save_busy_results: false + consumer: + fetch_max_messages: 5 + queues: + type: indexing_merge + when: + cluster_available: ["$[[INGEST_CLUSTER_ID]]"] + +#MANAGED_CONFIG_VERSION: 16 +#MANAGED: true \ No newline at end of file diff --git a/config_repo/templates/task_config.tpl b/config_repo/templates/task_config.tpl new file mode 100644 index 00000000..119d2fc8 --- /dev/null +++ b/config_repo/templates/task_config.tpl @@ -0,0 +1,87 @@ +elasticsearch: + - id: $[[CLUSTER_ID]] + name: $[[CLUSTER_ID]] + enabled: true + endpoints: $[[CLUSTER_ENDPOINT]] + discovery: + enabled: false + basic_auth: + username: $[[CLUSTER_USERNAME]] + password: $[[keystore.$[[CLUSTER_ID]]_password]] + +pipeline: +#clsuter level metrics +- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + keep_running: true + singleton: true + name: collect_$[[CLUSTER_ID]]_es_cluster_stats + retry_delay_in_ms: 10000 + processor: + - es_cluster_stats: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + when: + cluster_available: ["$[[CLUSTER_ID]]"] + +- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + keep_running: true + singleton: true + name: collect_$[[CLUSTER_ID]]_es_index_stats + retry_delay_in_ms: 10000 + processor: + - es_index_stats: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + when: + cluster_available: ["$[[CLUSTER_ID]]"] + +- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + keep_running: true + singleton: true + name: collect_$[[CLUSTER_ID]]_es_cluster_health + retry_delay_in_ms: 10000 + processor: + - es_cluster_health: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + when: + cluster_available: ["$[[CLUSTER_ID]]"] + +#node level metrics +- auto_start: $[[NODE_LEVEL_TASKS_ENABLED]] + enabled: $[[NODE_LEVEL_TASKS_ENABLED]] + keep_running: true + name: collect_$[[CLUSTER_ID]]_es_node_stats + retry_delay_in_ms: 10000 + processor: + - es_node_stats: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + when: + cluster_available: ["$[[CLUSTER_ID]]"] + +#node logs +- auto_start: $[[NODE_LEVEL_TASKS_ENABLED]] + enabled: $[[NODE_LEVEL_TASKS_ENABLED]] + keep_running: true + name: collect_$[[CLUSTER_ID]]_es_logs + retry_delay_in_ms: 10000 + processor: + - es_logs_processor: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + logs_path: $[[NODE_LOGS_PATH]] + queue_name: logs + when: + cluster_available: ["$[[CLUSTER_ID]]"] + +#MANAGED_CONFIG_VERSION: 11 +#MANAGED: true \ No newline at end of file diff --git a/console.yml b/console.yml index b10556bd..d1aa5231 100644 --- a/console.yml +++ b/console.yml @@ -1,6 +1,3 @@ -path.configs: "config" -configs.auto_reload: true - #env: # INFINI_CONSOLE_ENDPOINT: "http://127.0.0.1:9000" # INGEST_CLUSTER_ENDPOINT: "https://127.0.0.1:9200" @@ -11,6 +8,20 @@ configs.auto_reload: true # WECOM_WEBHOOK_ENDPOINT: # FEISHU_WEBHOOK_ENDPOINT: + +# must in major config file +path.configs: "config" +configs: + managed: true + auto_reload: true + manager: + local_configs_repo_path: ./config_repo/ + tls: #for mTLS connection with config servers + enabled: true + ca_file: config/certs/ca.crt + cert_file: config/certs/ca.crt + key_file: config/certs/ca.key + skip_insecure_verify: false web: enabled: true embedding_api: true @@ -62,6 +73,7 @@ badger: value_log_max_entries: 1000000 value_log_file_size: 104857600 + security: enabled: true # authc: diff --git a/main.go b/main.go index dd01f93e..095ffbba 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,7 @@ import ( "context" "errors" _ "expvar" + api3 "infini.sh/console/modules/agent/api" "infini.sh/console/plugin/api/email" model2 "infini.sh/framework/core/model" _ "time/tzdata" @@ -35,9 +36,9 @@ import ( "infini.sh/framework/modules/task" "infini.sh/framework/modules/ui" _ "infini.sh/framework/plugins" + _ "infini.sh/framework/plugins/managed" api2 "infini.sh/gateway/api" _ "infini.sh/gateway/proxy" - _ "infini.sh/framework/plugins/managed" ) var appConfig *config.AppConfig @@ -84,7 +85,7 @@ func main() { if !global.Env().SetupRequired() { for _, v := range modules { - module.RegisterModuleWithPriority(v.Value,v.Priority) + module.RegisterModuleWithPriority(v.Value, v.Priority) } } else { for _, v := range modules { @@ -94,6 +95,8 @@ func main() { api.RegisterAPI("") + api3.Init() + appConfig = &config.AppConfig{ UI: config.UIConfig{ LocalPath: ".public", @@ -103,7 +106,7 @@ func main() { } ok, err := env.ParseConfig("web", appConfig) - if err != nil { + if err != nil && global.Env().SystemConfig.Configs.PanicOnConfigError { panic(err) } if !ok { diff --git a/model/email_server.go b/model/email_server.go index 6c15545d..cc88dda5 100644 --- a/model/email_server.go +++ b/model/email_server.go @@ -6,7 +6,7 @@ package model import ( "fmt" - "infini.sh/framework/core/elastic" + "infini.sh/framework/core/model" "infini.sh/framework/core/orm" ) @@ -16,7 +16,7 @@ type EmailServer struct { Host string `json:"host" elastic_mapping:"host:{type:keyword}"` Port int `json:"port" elastic_mapping:"port:{type:keyword}"` TLS bool `json:"tls" elastic_mapping:"tls:{type:keyword}"` - Auth *elastic.BasicAuth `json:"auth" elastic_mapping:"auth:{type:object}"` + Auth *model.BasicAuth `json:"auth" elastic_mapping:"auth:{type:object}"` Enabled bool `json:"enabled" elastic_mapping:"enabled:{type:boolean}"` CredentialID string `json:"credential_id" elastic_mapping:"credential_id:{type:keyword}"` } diff --git a/modules/agent/api/elasticsearch.go b/modules/agent/api/elasticsearch.go new file mode 100644 index 00000000..dc5dfb4c --- /dev/null +++ b/modules/agent/api/elasticsearch.go @@ -0,0 +1,353 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package api + +import ( + "context" + "fmt" + log "github.com/cihub/seelog" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "net/http" + "time" +) + +//func (h *APIHandler) refreshESNodesInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +// id := ps.MustGetParameter("instance_id") +// obj := model.Instance{} +// obj.ID = id +// exists, err := orm.Get(&obj) +// if !exists || err != nil { +// h.WriteJSON(w, util.MapStr{ +// "_id": id, +// "found": false, +// }, http.StatusNotFound) +// return +// } +// _, err = refreshNodesInfo(&obj) +// if err != nil { +// log.Error(err) +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// return +// } +// h.WriteAckOKJSON(w) +//} + +func refreshNodesInfo(inst *model.Instance) ([]*model.ESNodeInfo, error) { + oldNodesInfo, err := getNodesBindingToAgent(inst) + if err != nil { + return nil, fmt.Errorf("error on get binding nodes info: %w", err) + } + + log.Error("oldNodesInfo:",util.MustToJSON(oldNodesInfo)) + + ctxTimeout, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + nodesInfo, err := GetElasticsearchNodesViaAgent(ctxTimeout, inst) + if err != nil { + //TODO return already biding nodes info ?? + return nil, fmt.Errorf("error on get nodes info from agent: %w", err) + } + + log.Error("nodesInfo:",util.MustToJSON(nodesInfo)) + + for _, node := range nodesInfo { + v,ok:=oldNodesInfo[node.NodeUUID] + if ok{ + node.ClusterID=v.ClusterID + } + } + + return nodesInfo, nil +} + +//get nodes info via agent +func GetElasticsearchNodesViaAgent(ctx context.Context, instance *model.Instance) ([]*model.ESNodeInfo, error) { + req := &util.Request{ + Method: http.MethodGet, + Path: "/elasticsearch/nodes/_discovery", + Context: ctx, + } + resBody := []*model.ESNodeInfo{} + err := doRequest(instance, req, &resBody) + if err != nil { + return nil, err + } + + return resBody, nil +} + +func AuthESNode(ctx context.Context, agentBaseURL string, cfg elastic.ElasticsearchConfig) (*model.ESNodeInfo, error) { + reqBody, err := util.ToJSONBytes(cfg) + if err != nil { + return nil, err + } + req := &util.Request{ + Method: http.MethodPost, + Path: "/elasticsearch/_auth", + Context: ctx, + Body: reqBody, + } + resBody := &model.ESNodeInfo{} + err = DoRequest(req, resBody) + if err != nil { + return nil, err + } + return resBody, nil +} + +func getNodeByPidOrUUID(nodes map[int]*model.ESNodeInfo, pid int, uuid string, port string) *model.ESNodeInfo { + if nodes[pid] != nil { + return nodes[pid] + } + for _, node := range nodes { + if node.NodeUUID != "" && node.NodeUUID == uuid { + return node + } + } + return nil +} + +type BindingItem struct { + ClusterID string `json:"cluster_id"` + ClusterUUID string `json:"cluster_uuid"` + NodeUUID string `json:"node_uuid"` +} + +//node -> binding item +func getNodesBindingToAgent(instance *model.Instance) (map[string]BindingItem, error) { + + //get nodes settings where agent id = instance id + q := orm.Query{ + Size: 1000, + Conds: orm.And(orm.Eq("metadata.category", "node_settings"), + orm.Eq("metadata.name", "agent"), + orm.Eq("metadata.labels.agent_id", instance.ID), + ), + } + + err, result := orm.Search(model.Setting{}, &q) + + if err != nil { + return nil, err + } + + ids := map[string]BindingItem{} + for _, row := range result.Result { + v, ok := row.(map[string]interface{}) + if ok { + x, ok := v["metadata"] + if ok { + y, ok := x.(map[string]interface{}) + if ok { + e, ok := y["labels"] + if ok { + f, ok := e.(map[string]interface{}) + if ok { + nodeID, ok := f["node_uuid"].(string) + if ok { + item := BindingItem{} + item.ClusterID = f["cluster_id"].(string) + item.ClusterUUID = f["cluster_uuid"].(string) + item.NodeUUID = nodeID + ids[item.NodeUUID] = item + } + } + } + } + } + } + } + return ids, nil +} + +func getUnAssociateNodes() (map[string][]model.ESNodeInfo, error) { + query := util.MapStr{ + "size": 3000, + "query": util.MapStr{ + "bool": util.MapStr{ + "must_not": []util.MapStr{ + { + "exists": util.MapStr{ + "field": "cluster_id", + }, + }, + }, + }, + }, + } + q := orm.Query{ + RawQuery: util.MustToJSONBytes(query), + } + + err, result := orm.Search(model.ESNodeInfo{}, &q) + if err != nil { + return nil, err + } + nodesInfo := map[string][]model.ESNodeInfo{} + for _, row := range result.Result { + node := model.ESNodeInfo{} + buf := util.MustToJSONBytes(row) + util.MustFromJSONBytes(buf, &node) + nodesInfo[node.AgentID] = append(nodesInfo[node.AgentID], node) + } + return nodesInfo, nil +} + +func GetElasticLogFiles(ctx context.Context, instance *model.Instance, logsPath string) (interface{}, error) { + + reqBody := util.MustToJSONBytes(util.MapStr{ + "logs_path": logsPath, + }) + req := &util.Request{ + Method: http.MethodPost, + Path: "/elasticsearch/logs/_list", + Context: ctx, + Body: reqBody, + } + resBody := map[string]interface{}{} + err := doRequest(instance, req, &resBody) + if err != nil { + return nil, err + } + if resBody["success"] != true { + return nil, fmt.Errorf("get elasticsearch log files error: %v", resBody) + } + return resBody["result"], nil +} + +func GetElasticLogFileContent(ctx context.Context, instance *model.Instance, body interface{}) (interface{}, error) { + req := &util.Request{ + Method: http.MethodPost, + Path: "/elasticsearch/logs/_read", + Context: ctx, + Body: util.MustToJSONBytes(body), + } + resBody := map[string]interface{}{} + err := doRequest(instance, req, &resBody) + if err != nil { + return nil, err + } + if resBody["success"] != true { + return nil, fmt.Errorf("get elasticsearch log files error: %v", resBody["error"]) + } + var hasMore bool + if v, ok := resBody["EOF"].(bool); ok && !v { + hasMore = true + } + return map[string]interface{}{ + "lines": resBody["result"], + "has_more": hasMore, + }, nil +} + +func (h *APIHandler) getLogFilesByNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + nodeID := ps.MustGetParameter("node_id") + inst, node, err := getAgentByNodeID(nodeID) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + if inst == nil { + log.Error(fmt.Sprintf("can not find agent by node [%s]", nodeID)) + h.WriteJSON(w, util.MapStr{ + "success": false, + "reason": "AGENT_NOT_FOUND", + }, http.StatusOK) + return + } + logFiles, err := GetElasticLogFiles(nil, inst, node.Path.Logs) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + h.WriteJSON(w, util.MapStr{ + "success": true, + "log_files": logFiles, + }, http.StatusOK) +} + +func (h *APIHandler) getLogFileContent(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + nodeID := ps.MustGetParameter("node_id") + inst, node, err := getAgentByNodeID(nodeID) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + if inst == nil { + h.WriteError(w, fmt.Sprintf("can not find agent by node [%s]", nodeID), http.StatusInternalServerError) + return + } + reqBody := struct { + FileName string `json:"file_name"` + LogsPath string `json:"logs_path"` + Offset int `json:"offset"` + Lines int `json:"lines"` + StartLineNumber int64 `json:"start_line_number"` + }{} + err = h.DecodeJSON(req, &reqBody) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + reqBody.LogsPath = node.Path.Logs + res, err := GetElasticLogFileContent(nil, inst, reqBody) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + h.WriteJSON(w, res, http.StatusOK) +} + +func getAgentByNodeID(nodeID string) (*model.Instance, *model.ESNodeInfo, error) { + queryDsl := util.MapStr{ + "size": 1, + "query": util.MapStr{ + "term": util.MapStr{ + "node_uuid": util.MapStr{ + "value": nodeID, + }, + }, + }, + "sort": []util.MapStr{ + { + "timestamp": util.MapStr{ + "order": "desc", + }, + }, + }, + } + q := &orm.Query{ + RawQuery: util.MustToJSONBytes(queryDsl), + } + err, result := orm.Search(model.ESNodeInfo{}, q) + if err != nil { + return nil, nil, err + } + if len(result.Result) > 0 { + buf := util.MustToJSONBytes(result.Result[0]) + v := &model.ESNodeInfo{} + err = util.FromJSONBytes(buf, v) + inst := &model.Instance{} + inst.ID = v.AgentID + _, err = orm.Get(inst) + if err != nil { + return nil, v, err + } + if inst.Name == "" { + return nil, v, nil + } + return inst, v, nil + } + return nil, nil, nil +} diff --git a/modules/agent/api/host.go b/modules/agent/api/host.go index f6341ebc..16419022 100644 --- a/modules/agent/api/host.go +++ b/modules/agent/api/host.go @@ -4,223 +4,219 @@ package api -// -//func (h *APIHandler) enrollHost(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { -// var reqBody []struct { -// AgentID string `json:"agent_id"` -// HostName string `json:"host_name"` -// IP string `json:"ip"` -// Source string `json:"source"` -// OSName string `json:"os_name"` -// OSArch string `json:"os_arch"` -// NodeID string `json:"node_uuid"` -// } -// err := h.DecodeJSON(req, &reqBody) -// if err != nil { -// log.Error(err) -// h.WriteError(w, err.Error(), http.StatusInternalServerError) -// return -// } -// errors := util.MapStr{} -// for i, hi := range reqBody { -// var ( -// hostInfo *host.HostInfo -// ) -// switch hi.Source { -// case "agent": -// hostInfo, err = enrollHostFromAgent(hi.AgentID) -// if err != nil { -// errors[hi.IP] = util.MapStr{ -// "error": err.Error(), -// } -// log.Error(err) -// continue -// } -// hostInfo.IP = hi.IP -// hostInfo.AgentID = hi.AgentID -// err = orm.Create(nil, hostInfo) -// if err != nil { -// errors[hi.IP] = util.MapStr{ -// "error": err.Error(), -// } -// log.Error(err) -// continue -// } -// case "es_node": -// hostInfo = &host.HostInfo{ -// IP: hi.IP, -// OSInfo: host.OS{ -// Platform: hi.OSName, -// KernelArch: hi.OSArch, -// }, -// NodeID: hi.NodeID, -// } -// default: -// errors[hi.IP] = util.MapStr{ -// "error": fmt.Errorf("unkonow source type"), -// } -// continue -// } -// hostInfo.Timestamp = time.Now() -// var ctx *orm.Context -// if i == len(reqBody) - 1 { -// ctx = &orm.Context{ -// Refresh: "wait_for", -// } -// } -// hostInfo.OSInfo.Platform = strings.ToLower(hostInfo.OSInfo.Platform) -// err = orm.Create(ctx, hostInfo) -// if err != nil { -// errors[hi.IP] = util.MapStr{ -// "error": err.Error(), -// } -// log.Error(err) -// continue -// } -// } -// resBody := util.MapStr{ -// "success": true, -// } -// if len(errors) > 0 { -// resBody["errors"] = errors -// resBody["success"] = false -// } -// -// h.WriteJSON(w, resBody, http.StatusOK) -//} -// -//func (h *APIHandler) deleteHost(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { -// hostID := ps.MustGetParameter("host_id") -// hostInfo, err := getHost(hostID) -// if err != nil { -// log.Error(err) -// h.WriteError(w, err.Error(), http.StatusInternalServerError) -// return -// } -// ctx := orm.Context{ -// Refresh: "wait_for", -// } -// err = orm.Delete(&ctx, hostInfo) -// if err != nil { -// log.Error(err) -// h.WriteError(w, err.Error(), http.StatusInternalServerError) -// return -// } -// h.WriteDeletedOKJSON(w, hostID) -//} -// -//func (h *APIHandler) GetHostAgentInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { -// hostID := ps.MustGetParameter("host_id") -// hostInfo, err := getHost(hostID) -// if err != nil { -// log.Error(err) -// h.WriteError(w, err.Error(), http.StatusInternalServerError) -// return -// } -// if hostInfo.AgentID == "" { -// h.WriteJSON(w, util.MapStr{}, http.StatusOK) -// return -// } -// -// sm := state.GetStateManager() -// ag, err := sm.GetAgent(hostInfo.AgentID) -// if err != nil { -// log.Error(err) -// h.WriteJSON(w, util.MapStr{}, http.StatusOK) -// return -// } -// aversion, err := ag.GetVersion() -// if err == nil { -// ag.Version = aversion -// orm.Save(nil, ag) -// } -// h.WriteJSON(w, util.MapStr{ -// "host_id": hostID, -// "agent_id": ag.ID, -// "version": ag.Version, -// "status": hostInfo.AgentStatus, -// "endpoint": ag.GetEndpoint(), -// }, http.StatusOK) -//} -// -//func getHost(hostID string) (*host.HostInfo, error){ -// hostInfo := &host.HostInfo{} -// hostInfo.ID = hostID -// exists, err := orm.Get(hostInfo) -// if err != nil { -// return nil, fmt.Errorf("get host info error: %w", err) -// } -// if !exists { -// return nil, fmt.Errorf("host [%s] not found", hostID) -// } -// return hostInfo, nil -//} -// -//func (h *APIHandler) GetHostElasticProcess(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { -// hostID := ps.MustGetParameter("host_id") -// hostInfo := &host.HostInfo{} -// hostInfo.ID = hostID -// exists, err := orm.Get(hostInfo) -// if err != nil { -// log.Error(err) -// h.WriteError(w, err.Error(), http.StatusInternalServerError) -// return -// } -// if !exists { -// h.WriteError(w, fmt.Sprintf("host [%s] not found", hostID), http.StatusNotFound) -// return -// } -// if hostInfo.AgentID == "" { -// h.WriteJSON(w, util.MapStr{}, http.StatusOK) -// return -// } -// sm := state.GetStateManager() -// ag, err := sm.GetAgent(hostInfo.AgentID) -// if err != nil { -// log.Error(err) -// h.WriteJSON(w, util.MapStr{}, http.StatusOK) -// return -// } -// ctx,cancel := context.WithTimeout(context.Background(), time.Second * 10) -// defer cancel() -// esNodesInfo, err := sm.GetAgentClient().GetElasticsearchNodes(ctx, ag.GetEndpoint()) -// if err != nil { -// log.Error(err) -// h.WriteError(w, err.Error(), http.StatusInternalServerError) -// return -// } -// var processes []util.MapStr -// for _, node := range esNodesInfo { -// processes = append(processes, util.MapStr{ -// "pid": node.ProcessInfo.PID, -// "pid_status": node.ProcessInfo.Status, -// "cluster_name": node.ClusterName, -// "cluster_uuid": node.ClusterUuid, -// "cluster_id": node.ClusterID, -// "node_id": node.NodeUUID, -// "node_name": node.NodeName, -// "uptime_in_ms": time.Now().UnixMilli() - node.ProcessInfo.CreateTime, -// }) -// } -// h.WriteJSON(w, util.MapStr{ -// "elastic_processes": processes, -// }, http.StatusOK) -//} -// -//func enrollHostFromAgent(agentID string) (*host.HostInfo, error){ -// sm := state.GetStateManager() -// ag, err := sm.GetAgent(agentID) -// if err != nil { -// return nil, err -// } -// if ag == nil { -// return nil, fmt.Errorf("can not found agent [%s]", agentID) -// } -// agentClient := sm.GetAgentClient() -// hostInfo, err := agentClient.GetHostInfo(nil, ag.GetEndpoint()) -// if err != nil { -// return nil, err -// } -// hostInfo.AgentStatus = ag.Status -// return hostInfo, nil -//} \ No newline at end of file +import ( + "context" + "fmt" + log "github.com/cihub/seelog" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/host" + "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "net/http" + "strings" + "time" +) + +func (h *APIHandler) enrollHost(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var reqBody []struct { + AgentID string `json:"agent_id"` + HostName string `json:"host_name"` + IP string `json:"ip"` + Source string `json:"source"` + OSName string `json:"os_name"` + OSArch string `json:"os_arch"` + NodeID string `json:"node_uuid"` + } + err := h.DecodeJSON(req, &reqBody) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + errors := util.MapStr{} + for i, hi := range reqBody { + var ( + hostInfo *host.HostInfo + ) + switch hi.Source { + case "agent": + obj := model.Instance{} + obj.ID = hi.AgentID + exists, err := orm.Get(&obj) + if !exists || err != nil { + continue + } + hostInfo = &host.HostInfo{} + hostInfo.IP = hi.IP + hostInfo.AgentID = hi.AgentID + err = orm.Create(nil, hostInfo) + if err != nil { + errors[hi.IP] = util.MapStr{ + "error": err.Error(), + } + log.Error(err) + continue + } + case "es_node": + hostInfo = &host.HostInfo{ + IP: hi.IP, + OSInfo: host.OS{ + Platform: hi.OSName, + KernelArch: hi.OSArch, + }, + NodeID: hi.NodeID, + } + default: + errors[hi.IP] = util.MapStr{ + "error": fmt.Errorf("unkonow source type"), + } + continue + } + hostInfo.Timestamp = time.Now() + var ctx *orm.Context + if i == len(reqBody) - 1 { + ctx = &orm.Context{ + Refresh: "wait_for", + } + } + hostInfo.OSInfo.Platform = strings.ToLower(hostInfo.OSInfo.Platform) + err = orm.Create(ctx, hostInfo) + if err != nil { + errors[hi.IP] = util.MapStr{ + "error": err.Error(), + } + log.Error(err) + continue + } + } + resBody := util.MapStr{ + "success": true, + } + if len(errors) > 0 { + resBody["errors"] = errors + resBody["success"] = false + } + + h.WriteJSON(w, resBody, http.StatusOK) +} + +func (h *APIHandler) deleteHost(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + hostID := ps.MustGetParameter("host_id") + hostInfo, err := getHost(hostID) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + ctx := orm.Context{ + Refresh: "wait_for", + } + err = orm.Delete(&ctx, hostInfo) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.WriteDeletedOKJSON(w, hostID) +} + +func (h *APIHandler) GetHostAgentInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + hostID := ps.MustGetParameter("host_id") + hostInfo, err := getHost(hostID) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if hostInfo.AgentID == "" { + h.WriteJSON(w, util.MapStr{}, http.StatusOK) + return + } + + obj := model.Instance{} + obj.ID = hostInfo.AgentID + exists, err := orm.Get(&obj) + if !exists || err != nil { + h.WriteJSON(w, util.MapStr{ + "_id": hostInfo.AgentID, + "found": false, + }, http.StatusNotFound) + return + } + + h.WriteJSON(w, util.MapStr{ + "host_id": hostID, + "agent_id": obj.ID, + "version": obj.Application.Version, + "status": hostInfo.AgentStatus, + "endpoint": obj.GetEndpoint(), + }, http.StatusOK) +} + +func getHost(hostID string) (*host.HostInfo, error){ + hostInfo := &host.HostInfo{} + hostInfo.ID = hostID + exists, err := orm.Get(hostInfo) + if err != nil { + return nil, fmt.Errorf("get host info error: %w", err) + } + if !exists { + return nil, fmt.Errorf("host [%s] not found", hostID) + } + return hostInfo, nil +} + +func (h *APIHandler) GetHostElasticProcess(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + hostID := ps.MustGetParameter("host_id") + hostInfo := &host.HostInfo{} + hostInfo.ID = hostID + exists, err := orm.Get(hostInfo) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if !exists { + h.WriteError(w, fmt.Sprintf("host [%s] not found", hostID), http.StatusNotFound) + return + } + if hostInfo.AgentID == "" { + h.WriteJSON(w, util.MapStr{}, http.StatusOK) + return + } + + obj := model.Instance{} + obj.ID = hostInfo.AgentID + exists, err = orm.Get(&obj) + if !exists || err != nil { + h.WriteJSON(w, util.MapStr{ + "_id": hostInfo.AgentID, + "found": false, + }, http.StatusNotFound) + return + } + + esNodesInfo, err := GetElasticsearchNodesViaAgent(context.Background(), &obj) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + var processes []util.MapStr + for _, node := range esNodesInfo { + processes = append(processes, util.MapStr{ + "pid": node.ProcessInfo.PID, + "pid_status": node.ProcessInfo.Status, + "cluster_name": node.ClusterName, + "cluster_uuid": node.ClusterUuid, + "cluster_id": node.ClusterID, + "node_id": node.NodeUUID, + "node_name": node.NodeName, + "uptime_in_ms": time.Now().UnixMilli() - node.ProcessInfo.CreateTime, + }) + } + h.WriteJSON(w, util.MapStr{ + "elastic_processes": processes, + }, http.StatusOK) +} \ No newline at end of file diff --git a/modules/agent/api/init.go b/modules/agent/api/init.go index 254d25f7..209c514a 100644 --- a/modules/agent/api/init.go +++ b/modules/agent/api/init.go @@ -4,32 +4,29 @@ package api -func Init() { - //handler := APIHandler{} - //api.HandleAPIMethod(api.POST, "/instance", handler.registerInstance) //new +import ( + "infini.sh/framework/core/api" + "infini.sh/framework/core/api/rbac/enum" +) + +func Init() { + handler := APIHandler{} + api.HandleAPIMethod(api.POST, "/host/_enroll", handler.enrollHost) + api.HandleAPIMethod(api.GET, "/host/:host_id/agent/info", handler.GetHostAgentInfo) + api.HandleAPIMethod(api.GET, "/host/:host_id/processes", handler.GetHostElasticProcess) + api.HandleAPIMethod(api.DELETE, "/host/:host_id", handler.deleteHost) + + api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_associate", handler.RequirePermission(handler.associateESNode, enum.PermissionAgentInstanceWrite)) + api.HandleAPIMethod(api.POST, "/auto_associate", handler.RequirePermission(handler.autoAssociateESNode, enum.PermissionAgentInstanceWrite)) + + //api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_auth", handler.RequirePermission(handler.authESNode, enum.PermissionAgentInstanceWrite)) + api.HandleAPIMethod(api.DELETE, "/instance/:instance_id/_nodes", handler.RequirePermission(handler.deleteESNode, enum.PermissionAgentInstanceWrite)) + api.HandleAPIMethod(api.GET, "/instance/:instance_id/_nodes", handler.RequirePermission(handler.getESNodesInfo, enum.PermissionAgentInstanceRead)) + + //get elasticsearch node logs, direct fetch or via stored logs(TODO) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/node/:node_id/logs/_list", handler.RequirePermission(handler.getLogFilesByNode, enum.PermissionAgentInstanceRead)) + api.HandleAPIMethod(api.POST, "/elasticsearch/:id/node/:node_id/logs/_read", handler.RequirePermission(handler.getLogFileContent, enum.PermissionAgentInstanceRead)) - //api.HandleAPIMethod(api.POST, "/agent/instance", handler.registerInstance) - //api.HandleAPIMethod(api.GET, "/agent/instance/_search", handler.RequirePermission(handler.searchInstance, enum.PermissionAgentInstanceRead)) - //api.HandleAPIMethod(api.GET, "/agent/instance/:instance_id", handler.getInstance) - //api.HandleAPIMethod(api.PUT, "/agent/instance/:instance_id", handler.updateInstance) - //api.HandleAPIMethod(api.DELETE, "/agent/instance/:instance_id", handler.RequirePermission(handler.deleteInstance, enum.PermissionAgentInstanceWrite)) - //api.HandleAPIMethod(api.POST, "/agent/instance/_stats", handler.RequirePermission(handler.getInstanceStats, enum.PermissionAgentInstanceRead)) - //api.HandleAPIMethod(api.GET, "/agent/log/node/:node_id/files", handler.RequirePermission(handler.getLogFilesByNode, enum.PermissionAgentInstanceRead)) - //api.HandleAPIMethod(api.POST, "/agent/log/node/:node_id/_scroll", handler.RequirePermission(handler.getLogFileContent, enum.PermissionAgentInstanceRead)) - //api.HandleAPIMethod(api.GET, "/agent/instance/:instance_id/_nodes", handler.RequirePermission(handler.getESNodesInfo, enum.PermissionAgentInstanceRead)) - //api.HandleAPIMethod(api.POST, "/agent/instance/:instance_id/_nodes/_refresh", handler.RequirePermission(handler.refreshESNodesInfo, enum.PermissionAgentInstanceWrite)) - //api.HandleAPIMethod(api.POST, "/agent/instance/:instance_id/node/_auth", handler.RequirePermission(handler.authESNode, enum.PermissionAgentInstanceWrite)) - //api.HandleAPIMethod(api.DELETE, "/agent/instance/:instance_id/_nodes", handler.RequirePermission(handler.deleteESNode, enum.PermissionAgentInstanceWrite)) - //api.HandleAPIMethod(api.POST, "/agent/instance/:instance_id/node/_associate", handler.RequirePermission(handler.associateESNode, enum.PermissionAgentInstanceWrite)) - //api.HandleAPIMethod(api.POST, "/agent/instance/try_connect", handler.RequireLogin(handler.tryConnect)) - //api.HandleAPIMethod(api.POST, "/agent/auto_associate", handler.RequirePermission(handler.autoAssociateESNode, enum.PermissionAgentInstanceWrite)) - // - //api.HandleAPIMethod(api.POST, "/host/_enroll", handler.enrollHost) - //api.HandleAPIMethod(api.GET, "/host/:host_id/agent/info",handler.GetHostAgentInfo) - //api.HandleAPIMethod(api.GET, "/host/:host_id/processes",handler.GetHostElasticProcess) - //api.HandleAPIMethod(api.DELETE, "/host/:host_id",handler.deleteHost) - // - // //api.HandleAPIMethod(api.POST, "/agent/install_command", handler.RequireLogin(handler.generateInstallCommand)) //api.HandleAPIMethod(api.GET, "/agent/install.sh", handler.getInstallScript) } diff --git a/modules/agent/api/instance.go b/modules/agent/api/instance.go index a6ebe0ed..563ce8cf 100644 --- a/modules/agent/api/instance.go +++ b/modules/agent/api/instance.go @@ -11,3 +11,4 @@ import ( type APIHandler struct { api.Handler } + diff --git a/modules/agent/api/log.go b/modules/agent/api/log.go deleted file mode 100644 index e4bb6341..00000000 --- a/modules/agent/api/log.go +++ /dev/null @@ -1,124 +0,0 @@ -/* Copyright © INFINI Ltd. All rights reserved. - * Web: https://infinilabs.com - * Email: hello#infini.ltd */ - -package api - -import ( - "fmt" - log "github.com/cihub/seelog" - "infini.sh/console/modules/agent/client" - "infini.sh/console/modules/agent/state" - httprouter "infini.sh/framework/core/api/router" - "infini.sh/framework/core/model" - "infini.sh/framework/core/orm" - "infini.sh/framework/core/util" - "net/http" -) - -func (h *APIHandler) getLogFilesByNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - nodeID := ps.MustGetParameter("node_id") - inst, node, err := getAgentByNodeID(nodeID) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - if inst == nil { - log.Error(fmt.Sprintf("can not find agent by node [%s]", nodeID)) - h.WriteJSON(w, util.MapStr{ - "success": false, - "reason": "AGENT_NOT_FOUND", - }, http.StatusOK) - return - } - logFiles, err := client.GetClient().GetElasticLogFiles(nil, inst.GetEndpoint(), node.Path.Logs) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - h.WriteJSON(w, util.MapStr{ - "success": true, - "log_files": logFiles, - }, http.StatusOK) -} - -func (h *APIHandler) getLogFileContent(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - nodeID := ps.MustGetParameter("node_id") - inst, node, err := getAgentByNodeID(nodeID) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - if inst == nil { - h.WriteError(w, fmt.Sprintf("can not find agent by node [%s]", nodeID), http.StatusInternalServerError) - return - } - reqBody := struct { - FileName string `json:"file_name"` - LogsPath string `json:"logs_path"` - Offset int `json:"offset"` - Lines int `json:"lines"` - StartLineNumber int64 `json:"start_line_number"` - }{} - err = h.DecodeJSON(req, &reqBody) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - reqBody.LogsPath = node.Path.Logs - sm := state.GetStateManager() - res, err := sm.GetAgentClient().GetElasticLogFileContent(nil, inst.GetEndpoint(), reqBody) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - h.WriteJSON(w, res, http.StatusOK) -} - -func getAgentByNodeID(nodeID string) (*model.Instance, *model.ESNodeInfo, error){ - queryDsl := util.MapStr{ - "size":1, - "query": util.MapStr{ - "term": util.MapStr{ - "node_uuid": util.MapStr{ - "value": nodeID, - }, - }, - }, - "sort": []util.MapStr{ - { - "timestamp": util.MapStr{ - "order": "desc", - }, - }, - }, - } - q := &orm.Query{ - RawQuery: util.MustToJSONBytes(queryDsl), - } - err, result := orm.Search(model.ESNodeInfo{}, q) - if err != nil { - return nil,nil, err - } - if len(result.Result) > 0 { - buf := util.MustToJSONBytes(result.Result[0]) - v := &model.ESNodeInfo{} - err = util.FromJSONBytes(buf, v) - inst := &model.Instance{} - inst.ID = v.AgentID - _, err = orm.Get(inst) - if err != nil { - return nil, v, err - } - if inst.Name == "" { - return nil, v, nil - } - return inst, v, nil - } - return nil, nil, nil -} diff --git a/modules/agent/api/tod.go b/modules/agent/api/tod.go new file mode 100644 index 00000000..72827ef4 --- /dev/null +++ b/modules/agent/api/tod.go @@ -0,0 +1,881 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package api + +import ( + "context" + "errors" + "fmt" + log "github.com/cihub/seelog" + common2 "infini.sh/console/modules/agent/common" + model3 "infini.sh/console/modules/agent/model" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/global" + "infini.sh/framework/core/host" + "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic/common" + "net/http" + "net/url" + "strings" + "sync" +) + +//func (h *APIHandler) updateInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +// id := ps.MustGetParameter("instance_id") +// oldInst := model.Instance{} +// oldInst.ID = id +// _, err := orm.Get(&oldInst) +// if err != nil { +// if err == elastic2.ErrNotFound { +// h.WriteJSON(w, util.MapStr{ +// "_id": id, +// "result": "not_found", +// }, http.StatusNotFound) +// return +// } +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// log.Error(err) +// return +// } +// +// obj := model.Instance{} +// err = h.DecodeJSON(req, &obj) +// if err != nil { +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// log.Error(err) +// return +// } +// +// oldInst.Name = obj.Name +// oldInst.Endpoint = obj.Endpoint +// oldInst.Description = obj.Description +// oldInst.Tags = obj.Tags +// oldInst.BasicAuth = obj.BasicAuth +// err = orm.Update(&orm.Context{ +// Refresh: "wait_for", +// }, &oldInst) +// if err != nil { +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// log.Error(err) +// return +// } +// +// h.WriteJSON(w, util.MapStr{ +// "_id": obj.ID, +// "result": "updated", +// }, 200) +//} +// +//func (h *APIHandler) searchInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +// +// var ( +// keyword = h.GetParameterOrDefault(req, "keyword", "") +// //queryDSL = `{"query":{"bool":{"must":[%s]}}, "size": %d, "from": %d}` +// strSize = h.GetParameterOrDefault(req, "size", "20") +// strFrom = h.GetParameterOrDefault(req, "from", "0") +// ) +// +// var ( +// mustQ []interface{} +// ) +// +// if keyword != "" { +// mustQ = append(mustQ, util.MapStr{ +// "query_string": util.MapStr{ +// "default_field": "*", +// "query": keyword, +// }, +// }) +// } +// size, _ := strconv.Atoi(strSize) +// if size <= 0 { +// size = 20 +// } +// from, _ := strconv.Atoi(strFrom) +// if from < 0 { +// from = 0 +// } +// +// queryDSL := util.MapStr{ +// "size": size, +// "from": from, +// } +// if len(mustQ) > 0 { +// queryDSL["query"] = util.MapStr{ +// "bool": util.MapStr{ +// "must": mustQ, +// }, +// } +// } +// +// q := orm.Query{} +// q.RawQuery = util.MustToJSONBytes(queryDSL) +// +// err, res := orm.Search(&model.Instance{}, &q) +// if err != nil { +// log.Error(err) +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// return +// } +// +// h.Write(w, res.Raw) +//} + +func (h *APIHandler) getESNodesInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + + id := ps.MustGetParameter("instance_id") + obj := model.Instance{} + obj.ID = id + exists, err := orm.Get(&obj) + if !exists || err != nil { + h.WriteJSON(w, util.MapStr{ + "_id": id, + "found": false, + }, http.StatusNotFound) + return + } + + nodes, err := refreshNodesInfo(&obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + h.WriteJSON(w, nodes, http.StatusOK) +} + +func (h *APIHandler) authESNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("instance_id") + inst := model.Instance{} + inst.ID = id + exists, err := orm.Get(&inst) + if !exists || err != nil { + h.WriteJSON(w, util.MapStr{ + "_id": id, + "found": false, + }, http.StatusNotFound) + return + } + reqBody := struct { + NodeID string `json:"node_id"` + ESConfig *elastic.ElasticsearchConfig `json:"es_config"` + }{} + err = h.DecodeJSON(req, &reqBody) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + //node id maybe is missing + if reqBody.NodeID != "" { + //verify the node id, if the node is is actually the node of the instance + oldNodeInfo := &model.ESNodeInfo{ + ID: reqBody.NodeID, + } + exists, err = orm.Get(oldNodeInfo) + if !exists || err != nil { + h.WriteJSON(w, fmt.Sprintf("node [%s] of agent [%s] was not found", oldNodeInfo.ID, inst.Name), http.StatusInternalServerError) + return + } + }else{ + //find out the node id with credentials + cfg := reqBody.ESConfig + if cfg.Endpoint == "" { + cfg.Endpoint = cfg.GetAnyEndpoint() + } + basicAuth, err := common.GetBasicAuth(cfg) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + cfg.BasicAuth = basicAuth + nodeInfo, err := AuthESNode(context.Background(), inst.GetEndpoint(), *cfg) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + //host, port, err := net.SplitHostPort(nodeInfo.PublishAddress) + //if err != nil { + // log.Error(err) + // h.WriteError(w, err.Error(), http.StatusInternalServerError) + // return + //} + //if !util.StringInArray(inst.Network.IP, host) && !net.ParseIP(host).IsLoopback() { + // h.WriteError(w, fmt.Sprintf("got node host %s not match any ip of %v", host, inst.Network.IP), http.StatusInternalServerError) + // return + //} + //if oldNodeInfo.HttpPort != port { + // h.WriteError(w, fmt.Sprintf("port mismatch, got: %s，expected: %s", port, oldNodeInfo.HttpPort), http.StatusInternalServerError) + // return + //} + //if oldNodeInfo.ProcessInfo.PID != nodeInfo.ProcessInfo.PID { + // h.WriteError(w, fmt.Sprintf("process id mismatch, got: %d，expected: %d", nodeInfo.ProcessInfo.PID, oldNodeInfo.ProcessInfo.PID), http.StatusInternalServerError) + // return + //} + + reqBody.NodeID=nodeInfo.NodeUUID + } + + + //nodeInfo:=elastic.NodeConfig{} + //nodeInfo.ID = reqBody.NodeID + //nodeInfo.AgentID = inst.ID + //err = orm.Update(nil, nodeInfo) //update node's info and agent_id + //if err != nil { + // log.Error(err) + // h.WriteError(w, err.Error(), http.StatusInternalServerError) + // return + //} + //h.WriteJSON(w, nodeInfo, http.StatusOK) +} + +func NewClusterSettings(clusterID string) *model.Setting { + settings := model.Setting{ + Metadata: model.SettingsMetadata{ + Category: Cluster, + }, + } + settings.ID = fmt.Sprintf("%v_%v_%v", settings.Metadata.Category, settings.Metadata.Name, clusterID) + + settings.Metadata.Labels = util.MapStr{ + "cluster_id": clusterID, + } + + return &settings +} + +func NewNodeAgentSettings(clusterID, clusterUUID, nodeUUID, agentID, agentCredential string) *model.Setting { + + settings := model.Setting{ + Metadata: model.SettingsMetadata{ + Category: Node, + Name: "agent", + }, + } + settings.ID = fmt.Sprintf("%v_%v_%v", settings.Metadata.Category, settings.Metadata.Name, nodeUUID) + + settings.Metadata.Labels = util.MapStr{ + "cluster_id": clusterID, + "cluster_uuid": clusterUUID, + "node_uuid": nodeUUID, + "agent_id": agentID, + "agent_credential": agentCredential, + } + + return &settings +} + +func NewIndexSettings(clusterID, nodeID, agentID, indexName, indexID string) *model.Setting { + + settings := model.Setting{ + Metadata: model.SettingsMetadata{ + Category: Index, + }, + } + settings.ID = fmt.Sprintf("%v_%v_%v", settings.Metadata.Category, settings.Metadata.Name, nodeID) + + settings.Metadata.Labels = util.MapStr{ + "cluster_id": clusterID, + "node_id": nodeID, + "agent_id": agentID, + "index_name": indexName, + "index_id": indexID, + } + + return &settings +} + +const Cluster = "cluster_settings" +const Node = "node_settings" +const Index = "index_settings" + +func (h *APIHandler) associateESNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + + //agent id + instID := ps.MustGetParameter("instance_id") + + //node id and cluster id + reqBody := struct { + ClusterUUID string `json:"cluster_uuid"` + NodeUUID string `json:"node_uuid"` + + //infini system assigned id + ClusterID string `json:"cluster_id"` + }{} + err := h.DecodeJSON(req, &reqBody) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + //update node's setting + settings := NewNodeAgentSettings(reqBody.ClusterID, reqBody.ClusterUUID, reqBody.NodeUUID, instID, "node.AgentCredential") + err = orm.Update(&orm.Context{ + Refresh: "wait_for", + }, settings) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + //settings, err := common2.GetAgentSettings(instID, 0) + //if err != nil { + // log.Error(err) + // h.WriteError(w, err.Error(), http.StatusInternalServerError) + // return + //} + //setting := pickAgentSettings(settings, node) + //if setting == nil { + // setting, err = getAgentTaskSetting(instID, node) + // if err != nil { + // log.Error("get agent task setting error: ", err) + // } + // err = orm.Create(nil, setting) + // if err != nil { + // log.Error("save agent task setting error: ", err) + // } + //} + + h.WriteAckOKJSON(w) +} + +func (h *APIHandler) autoAssociateESNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + reqBody := struct { + ClusterIDs []string `json:"cluster_ids"` + }{} + err := h.DecodeJSON(req, &reqBody) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + // query not associated nodes info + nodesM, err := getUnAssociateNodes() + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if len(nodesM) == 0 { + h.WriteAckOKJSON(w) + return + } + agentIds := make([]string, 0, len(nodesM)) + for agentID := range nodesM { + agentIds = append(agentIds, agentID) + } + agents, err := getAgentByIds(agentIds) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + for _, clusterID := range reqBody.ClusterIDs { + // query cluster basicauth + cfg := elastic.GetConfig(clusterID) + basicAuth, err := common.GetBasicAuth(cfg) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + taskSetting, err := getSettingsByClusterID(cfg.ID) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + for agentID, nodes := range nodesM { + var ( + inst *model.Instance + ok bool + ) + if inst, ok = agents[agentID]; !ok { + log.Warnf("agent [%v] was not found", agentID) + continue + } + settings, err := common2.GetAgentSettings(agentID, 0) + if err != nil { + log.Error(err) + continue + } + for _, v := range nodes { + host := v.PublishAddress + var endpoint string + if strings.HasPrefix(host, "::") { //for ipv6 + instURL, err := url.Parse(inst.Endpoint) + if err != nil { + log.Error(err) + continue + } + host = instURL.Hostname() + endpoint = fmt.Sprintf("%s://[%s]:%s", v.Schema, host, v.HttpPort) + } else { + endpoint = fmt.Sprintf("%s://%s", v.Schema, host) + } + escfg := elastic.ElasticsearchConfig{ + Endpoint: endpoint, + BasicAuth: basicAuth, + } + nodeInfo, err := AuthESNode(context.Background(), inst.GetEndpoint(), escfg) + if err != nil { + log.Warn(err) + continue + } + //matched + if nodeInfo.ClusterUuid == cfg.ClusterUUID { + //update node info + nodeInfo.ID = v.ID + nodeInfo.AgentID = inst.ID + nodeInfo.ClusterID = cfg.ID + err = orm.Save(nil, nodeInfo) + if err != nil { + log.Error(err) + continue + } + setting := pickAgentSettings(settings, v) + if setting == nil { + tsetting := model3.TaskSetting{ + NodeStats: &model3.NodeStatsTask{ + Enabled: true, + }, + Logs: &model3.LogsTask{ + Enabled: true, + LogsPath: nodeInfo.Path.Logs, + }, + } + if taskSetting.IndexStats != nil { + tsetting.IndexStats = taskSetting.IndexStats + taskSetting.IndexStats = nil + } + if taskSetting.ClusterHealth != nil { + tsetting.ClusterHealth = taskSetting.ClusterHealth + taskSetting.ClusterHealth = nil + } + if taskSetting.ClusterStats != nil { + tsetting.ClusterStats = taskSetting.ClusterStats + taskSetting.ClusterStats = nil + } + setting = &model.Setting{ + Metadata: model.SettingsMetadata{ + Category: "agent", + Name: "task", + Labels: util.MapStr{ + "agent_id": agentID, + "cluster_uuid": nodeInfo.ClusterUuid, + "cluster_id": nodeInfo.ClusterID, + "node_uuid": nodeInfo.NodeUUID, + "endpoint": fmt.Sprintf("%s://%s", nodeInfo.Schema, nodeInfo.PublishAddress), + }, + }, + Payload: util.MapStr{ + "task": tsetting, + }, + } + err = orm.Create(nil, setting) + if err != nil { + log.Error("save agent task setting error: ", err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + } + } + } + + } + } + h.WriteAckOKJSON(w) +} + +func getAgentByIds(agentIDs []string) (map[string]*model.Instance, error) { + query := util.MapStr{ + "size": len(agentIDs), + "query": util.MapStr{ + "terms": util.MapStr{ + "id": agentIDs, + }, + }, + } + q := orm.Query{ + RawQuery: util.MustToJSONBytes(query), + } + err, result := orm.Search(model.Instance{}, &q) + if err != nil { + return nil, err + } + agents := map[string]*model.Instance{} + for _, row := range result.Result { + inst := model.Instance{} + buf := util.MustToJSONBytes(row) + util.MustFromJSONBytes(buf, &inst) + agents[inst.ID] = &inst + } + return agents, nil +} + +func (h *APIHandler) deleteESNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("instance_id") + nodeIDs := []string{} + err := h.DecodeJSON(req, &nodeIDs) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if len(nodeIDs) > 0 { + q := util.MapStr{ + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "terms": util.MapStr{ + "id": nodeIDs, + }, + }, + { + "term": util.MapStr{ + "agent_id": util.MapStr{ + "value": id, + }, + }, + }, + }, + }, + }, + } + err = orm.DeleteBy(model.ESNodeInfo{}, util.MustToJSONBytes(q)) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + q = util.MapStr{ + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "terms": util.MapStr{ + "metadata.labels.node_uuid": nodeIDs, + }, + }, + { + "term": util.MapStr{ + "metadata.labels.agent_id": util.MapStr{ + "value": id, + }, + }, + }, + }, + }, + }, + } + } + h.WriteAckOKJSON(w) +} + +// +//func (h *APIHandler) tryConnect(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +// var reqBody = struct { +// Endpoint string `json:"endpoint"` +// BasicAuth model.BasicAuth +// }{} +// err := h.DecodeJSON(req, &reqBody) +// if err != nil { +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// return +// } +// connectRes, err := client.GetClient().GetInstanceBasicInfo(context.Background(), reqBody.Endpoint) +// if err != nil { +// h.WriteError(w, err.Error(), http.StatusInternalServerError) +// return +// } +// h.WriteJSON(w, connectRes, http.StatusOK) +//} + +func pickAgentSettings(settings []model.Setting, nodeInfo model.ESNodeInfo) *model.Setting { + for _, setting := range settings { + if setting.Metadata.Labels["node_uuid"] == nodeInfo.NodeUUID { + return &setting + } + } + return nil +} + +func getAgentTaskSetting(agentID string, v model.ESNodeInfo) (*model.Setting, error) { + taskSetting, err := getSettingsByClusterID(v.ClusterID) + if err != nil { + return nil, err + } + taskSetting.Logs = &model3.LogsTask{ + Enabled: true, + LogsPath: v.Path.Logs, + } + return &model.Setting{ + Metadata: model.SettingsMetadata{ + Category: "agent", + Name: "task", + Labels: util.MapStr{ + "agent_id": agentID, + "cluster_uuid": v.ClusterUuid, + "cluster_id": v.ClusterID, + "node_uuid": v.NodeUUID, + "endpoint": fmt.Sprintf("%s://%s", v.Schema, v.PublishAddress), + }, + }, + Payload: util.MapStr{ + "task": taskSetting, + }, + }, nil +} + +// getSettingsByClusterID query agent task settings with cluster id +func getSettingsByClusterID(clusterID string) (*model3.TaskSetting, error) { + err, result := querySettingsByClusterID(clusterID) + if err != nil { + return nil, err + } + + setting := &model3.TaskSetting{ + NodeStats: &model3.NodeStatsTask{ + Enabled: true, + }, + } + var ( + clusterStats = true + indexStats = true + clusterHealth = true + ) + keys := []string{"payload.task.cluster_stats.enabled", "payload.task.cluster_health.enabled", "payload.task.index_stats.enabled"} + for _, row := range result.Result { + if v, ok := row.(map[string]interface{}); ok { + vm := util.MapStr(v) + for _, key := range keys { + tv, _ := vm.GetValue(key) + if tv == true { + switch key { + case "payload.task.cluster_stats.enabled": + clusterStats = false + case "payload.task.index_stats.enabled": + indexStats = false + case "payload.task.cluster_health.enabled": + clusterHealth = false + } + } + } + } + } + if clusterStats { + setting.ClusterStats = &model3.ClusterStatsTask{ + Enabled: true, + } + } + if indexStats { + setting.IndexStats = &model3.IndexStatsTask{ + Enabled: true, + } + } + if clusterHealth { + setting.ClusterHealth = &model3.ClusterHealthTask{ + Enabled: true, + } + } + return setting, nil +} + +func querySettingsByClusterID(clusterID string) (error, orm.Result) { + queryDsl := util.MapStr{ + "size": 500, + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": clusterID, + }, + }, + }, + }, + "minimum_should_match": 1, + "should": []util.MapStr{ + { + "term": util.MapStr{ + "payload.task.cluster_health.enabled": util.MapStr{ + "value": true, + }, + }, + }, + { + "term": util.MapStr{ + "payload.task.cluster_stats.enabled": util.MapStr{ + "value": true, + }, + }, + }, + { + "term": util.MapStr{ + "payload.task.index_stats.enabled": util.MapStr{ + "value": true, + }, + }, + }, + }, + }, + }, + } + q := orm.Query{ + RawQuery: util.MustToJSONBytes(queryDsl), + } + return orm.Search(model.Setting{}, &q) +} + +func GetHostInfo(ctx context.Context, agentBaseURL string) (*host.HostInfo, error) { + req := &util.Request{ + Method: http.MethodGet, + Path: "/agent/host/_basic", + Context: ctx, + } + resBody := struct { + Success bool `json:"success"` + Error string `json:"error"` + HostInfo *host.HostInfo `json:"result"` + }{} + + req.Body = util.MustToJSONBytes(resBody) + + err := DoRequest(req, &resBody) + if err != nil { + return nil, err + } + + if resBody.Success != true { + return nil, fmt.Errorf("enroll error from client: %v", resBody.Error) + } + return resBody.HostInfo, nil +} + +func GetElasticProcess(ctx context.Context, agentBaseURL string, agentID string) (interface{}, error) { + req := &util.Request{ + Method: http.MethodGet, + Path: fmt.Sprintf("/elasticsearch/%s/process/_elastic", agentID), + Context: ctx, + } + resBody := map[string]interface{}{} + err := DoRequest(req, &resBody) + if err != nil { + return nil, err + } + if resBody["success"] != true { + return nil, fmt.Errorf("discover host callback error: %v", resBody["error"]) + } + return resBody["elastic_process"], nil +} + +func GetInstanceBasicInfo(ctx context.Context, agentBaseURL string) (*model.Instance, error) { + req := &util.Request{ + Method: http.MethodGet, + Path: "/_info", + Context: ctx, + } + resBody := &model.Instance{} + err := DoRequest(req, &resBody) + return resBody, err +} + +func RegisterElasticsearch(ctx context.Context, agentBaseURL string, cfgs []elastic.ElasticsearchConfig) error { + reqBody, err := util.ToJSONBytes(cfgs) + if err != nil { + return err + } + req := &util.Request{ + Method: http.MethodPost, + Path: "/elasticsearch/_register", + Context: ctx, + Body: reqBody, + } + resBody := util.MapStr{} + err = DoRequest(req, &resBody) + if err != nil { + return err + } + if resBody["acknowledged"] != true { + return fmt.Errorf("%v", resBody["error"]) + } + return nil +} + +func CreatePipeline(ctx context.Context, agentBaseURL string, body []byte) error { + req := &util.Request{ + Method: http.MethodPost, + Path: "/pipeline/tasks/", + Body: body, + Context: ctx, + } + resBody := util.MapStr{} + return DoRequest(req, &resBody) +} + +func DeletePipeline(ctx context.Context, agentBaseURL, pipelineID string) error { + req := &util.Request{ + Method: http.MethodDelete, + Path: fmt.Sprintf("/pipeline/task/%s", pipelineID), + Context: ctx, + } + return DoRequest(req, nil) +} + +func DoRequest(req *util.Request, obj interface{}) error { + panic("implement me") +} + +var mTLSClient *http.Client //TODO get mTLSClient +var initOnce = sync.Once{} + +func doRequest(instance *model.Instance, req *util.Request, obj interface{}) error { + var err error + var res *util.Result + + initOnce.Do(func() { + if global.Env().SystemConfig.Configs.TLSConfig.TLSEnabled && global.Env().SystemConfig.Configs.TLSConfig.TLSCAFile != "" { + + //init client + hClient, err := util.NewMTLSClient( + global.Env().SystemConfig.Configs.TLSConfig.TLSCAFile, + global.Env().SystemConfig.Configs.TLSConfig.TLSCertFile, + global.Env().SystemConfig.Configs.TLSConfig.TLSKeyFile) + if err != nil { + panic(err) + } + mTLSClient = hClient + } + }) + + req.Url, err = url.JoinPath(instance.GetEndpoint(), req.Path) + res, err = util.ExecuteRequestWithCatchFlag(mTLSClient, req, true) + if err != nil || res.StatusCode != 200 { + body := "" + if res != nil { + body = string(res.Body) + } + return errors.New(fmt.Sprintf("request error: %v, %v", err, body)) + } + + if res != nil { + if res.Body != nil { + return util.FromJSONBytes(res.Body, obj) + } + } + + return nil +} diff --git a/modules/agent/client/client.go b/modules/agent/client/client.go deleted file mode 100644 index 7f26092d..00000000 --- a/modules/agent/client/client.go +++ /dev/null @@ -1,279 +0,0 @@ -/* Copyright © INFINI Ltd. All rights reserved. - * Web: https://infinilabs.com - * Email: hello#infini.ltd */ - -package client - -import ( - "context" - "fmt" - "infini.sh/console/modules/agent/common" - "infini.sh/framework/core/model" - "infini.sh/framework/core/elastic" - "infini.sh/framework/core/host" - "infini.sh/framework/core/util" - "net/http" -) - -var defaultClient ClientAPI - -func GetClient() ClientAPI { - if defaultClient == nil { - panic("agent client not init") - } - return defaultClient -} - -func RegisterClient(client ClientAPI) { - defaultClient = client -} -type ClientAPI interface { - GetHostInfo(ctx context.Context, agentBaseURL string) (*host.HostInfo, error) - GetElasticProcess(ctx context.Context, agentBaseURL string, agentID string)(interface{}, error) - GetElasticLogFiles(ctx context.Context, agentBaseURL string, logsPath string)(interface{}, error) - GetElasticLogFileContent(ctx context.Context, agentBaseURL string, body interface{})(interface{}, error) - GetInstanceBasicInfo(ctx context.Context, agentBaseURL string) (*model.Instance, error) - RegisterElasticsearch(ctx context.Context, agentBaseURL string, cfgs []elastic.ElasticsearchConfig) error - GetElasticsearchNodes(ctx context.Context, agentBaseURL string) ([]model.ESNodeInfo, error) - AuthESNode(ctx context.Context, agentBaseURL string, cfg elastic.ElasticsearchConfig) (*model.ESNodeInfo, error) - CreatePipeline(ctx context.Context, agentBaseURL string, body []byte) error - DeletePipeline(ctx context.Context, agentBaseURL, pipelineID string) error - SetKeystoreValue(ctx context.Context, agentBaseURL string, key, value string) error - SaveDynamicConfig(ctx context.Context, agentBaseURL string, name, content string) error - SaveIngestConfig(ctx context.Context, agentBaseURL string) error - DoRequest(req *util.Request, respObj interface{}) error -} - -type Client struct { - Executor Executor -} - - -//func (client *Client) GetHostInfo(ctx context.Context, agentBaseURL string) (*host.HostInfo, error) { -// req := &util.Request{ -// Method: http.MethodGet, -// Url: fmt.Sprintf("%s/agent/host/_basic", agentBaseURL), -// Context: ctx, -// } -// resBody := struct { -// Success bool `json:"success"` -// Error string `json:"error"` -// HostInfo *host.HostInfo `json:"result"` -// }{} -// err := client.DoRequest(req, &resBody) -// if err != nil { -// return nil, err -// } -// if resBody.Success != true { -// return nil, fmt.Errorf("enroll error from client: %v", resBody.Error) -// } -// return resBody.HostInfo, nil -//} - -//TODO -func (client *Client) GetElasticProcess(ctx context.Context, agentBaseURL string, agentID string)(interface{}, error) { - panic("implement me") - req := &util.Request{ - Method: http.MethodGet, - Url: fmt.Sprintf("%s/elasticsearch/%s/process/_elastic", agentBaseURL, agentID), - Context: ctx, - } - resBody := map[string]interface{}{} - err := client.DoRequest(req, &resBody) - if err != nil { - return nil, err - } - if resBody["success"] != true { - return nil, fmt.Errorf("discover host callback error: %v", resBody["error"]) - } - return resBody["elastic_process"], nil -} - -func (client *Client) GetElasticLogFiles(ctx context.Context, agentBaseURL string, logsPath string)(interface{}, error) { - panic("implement me") - - - reqBody := util.MustToJSONBytes(util.MapStr{ - "logs_path": logsPath, - }) - req := &util.Request{ - Method: http.MethodPost, - Url: fmt.Sprintf("%s/agent/logs/elastic/list", agentBaseURL), - Context: ctx, - Body: reqBody, - } - resBody := map[string]interface{}{} - err := client.DoRequest(req, &resBody) - if err != nil { - return nil, err - } - if resBody["success"] != true { - return nil, fmt.Errorf("get elasticsearch log files error: %v", resBody["error"]) - } - return resBody["result"], nil -} - -func (client *Client) GetElasticLogFileContent(ctx context.Context, agentBaseURL string, body interface{})(interface{}, error) { - req := &util.Request{ - Method: http.MethodPost, - Url: fmt.Sprintf("%s/agent/logs/elastic/_read", agentBaseURL), - Context: ctx, - Body: util.MustToJSONBytes(body), - } - resBody := map[string]interface{}{} - err := client.DoRequest(req, &resBody) - if err != nil { - return nil, err - } - if resBody["success"] != true { - return nil, fmt.Errorf("get elasticsearch log files error: %v", resBody["error"]) - } - var hasMore bool - if v, ok := resBody["EOF"].(bool); ok && !v { - hasMore = true - } - return map[string]interface{}{ - "lines": resBody["result"], - "has_more": hasMore, - } , nil -} - -func (client *Client) GetInstanceBasicInfo(ctx context.Context, agentBaseURL string) (*model.Instance, error){ - req := &util.Request{ - Method: http.MethodGet, - Url: fmt.Sprintf("%s/_info", agentBaseURL ), - Context: ctx, - } - resBody := &model.Instance{} - err := client.DoRequest(req, &resBody) - return resBody, err -} - -func (client *Client) RegisterElasticsearch(ctx context.Context, agentBaseURL string, cfgs []elastic.ElasticsearchConfig) error { - reqBody, err := util.ToJSONBytes(cfgs) - if err != nil { - return err - } - req := &util.Request{ - Method: http.MethodPost, - Url: fmt.Sprintf("%s/elasticsearch/_register", agentBaseURL ), - Context: ctx, - Body: reqBody, - } - resBody := util.MapStr{} - err = client.DoRequest(req, &resBody) - if err != nil { - return err - } - if resBody["acknowledged"] != true { - return fmt.Errorf("%v", resBody["error"]) - } - return nil -} - -func (client *Client) GetElasticsearchNodes(ctx context.Context, agentBaseURL string) ([]model.ESNodeInfo, error) { - req := &util.Request{ - Method: http.MethodGet, - Url: fmt.Sprintf("%s/elasticsearch/nodes/_discovery", agentBaseURL ), - Context: ctx, - } - resBody := []model.ESNodeInfo{} - err := client.DoRequest(req, &resBody) - if err != nil { - return nil, err - } - - return resBody, nil -} - -func (client *Client) AuthESNode(ctx context.Context, agentBaseURL string, cfg elastic.ElasticsearchConfig) (*model.ESNodeInfo, error) { - reqBody, err := util.ToJSONBytes(cfg) - if err != nil { - return nil, err - } - req := &util.Request{ - Method: http.MethodPost, - Url: fmt.Sprintf("%s/elasticsearch/_auth", agentBaseURL ), - Context: ctx, - Body: reqBody, - } - resBody := &model.ESNodeInfo{} - err = client.DoRequest(req, resBody) - if err != nil { - return nil, err - } - return resBody, nil -} - -func (client *Client) CreatePipeline(ctx context.Context, agentBaseURL string, body []byte) error{ - req := &util.Request{ - Method: http.MethodPost, - Url: agentBaseURL + "/pipeline/tasks/", - Body: body, - Context: ctx, - } - resBody := util.MapStr{} - return client.DoRequest(req, &resBody) -} - -func (client *Client) DeletePipeline(ctx context.Context, agentBaseURL, pipelineID string) error{ - req := &util.Request{ - Method: http.MethodDelete, - Url: fmt.Sprintf("%s/pipeline/task/%s", agentBaseURL, pipelineID), - Context: ctx, - } - return client.DoRequest(req, nil) -} - -func (client *Client) SetKeystoreValue(ctx context.Context, agentBaseURL string, key, value string) error{ - body := util.MapStr{ - "key": key, - "value": value, - } - req := &util.Request{ - Method: http.MethodPost, - Url: fmt.Sprintf("%s/keystore", agentBaseURL), - Context: ctx, - Body: util.MustToJSONBytes(body), - } - return client.DoRequest(req, nil) -} - -func (client *Client) SaveDynamicConfig(ctx context.Context, agentBaseURL string, filename, content string) error{ - body := util.MapStr{ - "configs": util.MapStr{ - filename: content, - }, - } - req := &util.Request{ - Method: http.MethodPost, - Url: fmt.Sprintf("%s/config/_update", agentBaseURL), - Context: ctx, - Body: util.MustToJSONBytes(body), - } - return client.DoRequest(req, nil) -} - -func (client *Client) SaveIngestConfig(ctx context.Context, agentBaseURL string) error { - ingestCfg, basicAuth, err := common.GetAgentIngestConfig() - if err != nil { - return err - } - if basicAuth != nil && basicAuth.Password != "" { - err = client.SetKeystoreValue(ctx, agentBaseURL, "ingest_cluster_password", basicAuth.Password) - if err != nil { - return fmt.Errorf("set keystore value to agent error: %w", err) - } - } - err = client.SaveDynamicConfig(context.Background(), agentBaseURL, "ingest_variables.yml", ingestCfg ) - if err != nil { - return fmt.Errorf("save dynamic config to agent error: %w", err) - } - return nil -} - - -func (client *Client) DoRequest(req *util.Request, respObj interface{}) error { - return client.Executor.DoRequest(req, respObj) -} - diff --git a/modules/agent/client/executor.go b/modules/agent/client/executor.go deleted file mode 100644 index f756f3a1..00000000 --- a/modules/agent/client/executor.go +++ /dev/null @@ -1,100 +0,0 @@ -/* Copyright © INFINI Ltd. All rights reserved. - * Web: https://infinilabs.com - * Email: hello#infini.ltd */ - -package client - -import ( - "bytes" - "fmt" - "infini.sh/console/modules/agent/common" - "infini.sh/framework/core/util" - "io" - "net/http" -) - -type Executor interface { - DoRequest(req *util.Request, respObj interface{}) error -} - -type HttpExecutor struct { -} - -func (executor *HttpExecutor) DoRequest(req *util.Request, respObj interface{}) error { - result, err := util.ExecuteRequest(req) - if err != nil { - return err - } - if result.StatusCode != 200 { - return fmt.Errorf(string(result.Body)) - } - if respObj == nil { - return nil - } - return util.FromJSONBytes(result.Body, respObj) -} - -func NewMTLSExecutor(caCertFile, caKeyFile string) (*MTLSExecutor, error){ - var ( - instanceCrt string - instanceKey string - ) - instanceCrt, instanceKey, err := common.GetAgentInstanceCerts(caCertFile, caKeyFile) - if err != nil { - return nil, fmt.Errorf("generate tls cert error: %w", err) - } - hClient, err := util.NewMTLSClient(caCertFile, instanceCrt, instanceKey) - if err != nil { - return nil, err - } - return &MTLSExecutor{ - CaCertFile: caCertFile, - CAKeyFile: caKeyFile, - client: hClient, - }, nil -} - -type MTLSExecutor struct { - CaCertFile string - CAKeyFile string - client *http.Client -} - - -func (executor *MTLSExecutor) DoRequest(req *util.Request, respObj interface{}) error { - var reader io.Reader - if len(req.Body) > 0 { - reader = bytes.NewReader(req.Body) - } - var ( - hr *http.Request - err error - ) - if req.Context == nil { - hr, err = http.NewRequest(req.Method, req.Url, reader) - }else{ - hr, err = http.NewRequestWithContext(req.Context, req.Method, req.Url, reader) - } - if err != nil { - return err - } - res, err := executor.client.Do(hr) - if err != nil { - return err - } - defer res.Body.Close() - buf, err := io.ReadAll(res.Body) - if err != nil { - return err - } - if res.StatusCode != 200 { - return fmt.Errorf(string(buf)) - } - if respObj != nil { - err = util.FromJSONBytes(buf, respObj) - if err != nil { - return err - } - } - return nil -} \ No newline at end of file diff --git a/modules/agent/common/config.go b/modules/agent/common/config.go index fb7e6ed9..3d1ccbf2 100644 --- a/modules/agent/common/config.go +++ b/modules/agent/common/config.go @@ -5,15 +5,10 @@ package common import ( - "crypto/x509" - "encoding/pem" log "github.com/cihub/seelog" "infini.sh/console/modules/agent/model" "infini.sh/framework/core/env" - "infini.sh/framework/core/global" - "infini.sh/framework/core/util" - "os" - "path" + "infini.sh/framework/plugins/managed/common" ) @@ -30,37 +25,10 @@ func GetAgentConfig() *model.AgentConfig { log.Debug("agent config not found: %v", err) } if agentCfg.Setup.CACertFile == "" && agentCfg.Setup.CAKeyFile == "" { - agentCfg.Setup.CACertFile, agentCfg.Setup.CAKeyFile, err = GetOrInitDefaultCaCerts() + agentCfg.Setup.CACertFile, agentCfg.Setup.CAKeyFile, err = common.GetOrInitDefaultCaCerts() if err != nil { log.Errorf("generate default ca certs error: %v", err) } } return agentCfg } - -func GetOrInitDefaultCaCerts()(string, string, error){ - dataDir := global.Env().GetDataDir() - caFile := path.Join(dataDir, "certs/ca.crt") - caKey := path.Join(dataDir, "certs/ca.key") - if !(util.FileExists(caFile) && util.FileExists(caKey) ) { - err := os.MkdirAll(path.Join(dataDir, "certs"), 0775) - if err != nil { - return "", "", err - } - log.Info("auto generating cert files") - _, rootKey, rootCertPEM := util.GetRootCert() - - caKeyPEM := pem.EncodeToMemory(&pem.Block{ - Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(rootKey), - }) - _, err = util.FilePutContentWithByte(caKey, caKeyPEM) - if err != nil { - return "", "", err - } - _, err = util.FilePutContentWithByte(caFile, rootCertPEM) - if err != nil { - return "", "", err - } - } - return caFile, caKey, nil -} \ No newline at end of file diff --git a/modules/agent/common/helper.go b/modules/agent/common/helper.go index 5f75db2a..7baa7952 100644 --- a/modules/agent/common/helper.go +++ b/modules/agent/common/helper.go @@ -401,7 +401,7 @@ func GetLatestOnlineAgentIDs(agentIds []string, lastSeconds int) (map[string]str return agentIDs, nil } -func GetAgentIngestConfig() (string, *elastic.BasicAuth, error) { +func GetAgentIngestConfig() (string, *model.BasicAuth, error) { agCfg := GetAgentConfig() var ( endpoint string @@ -422,7 +422,7 @@ func GetAgentIngestConfig() (string, *elastic.BasicAuth, error) { } var ( - basicAuth elastic.BasicAuth + basicAuth model.BasicAuth ) if agCfg.Setup.IngestClusterCredentialID != "" { cred := credential.Credential{} @@ -435,7 +435,7 @@ func GetAgentIngestConfig() (string, *elastic.BasicAuth, error) { if err != nil { return "", nil, fmt.Errorf("decode credential [%s] error: %w", cred.ID, err) } - if basicAuth, ok = info.(elastic.BasicAuth); !ok { + if basicAuth, ok = info.(model.BasicAuth); !ok { log.Debug("invalid credential: ", cred) } }else{ diff --git a/modules/agent/common/helper_test.go b/modules/agent/common/helper_test.go deleted file mode 100644 index 285df64a..00000000 --- a/modules/agent/common/helper_test.go +++ /dev/null @@ -1,41 +0,0 @@ -/* Copyright © INFINI Ltd. All rights reserved. - * Web: https://infinilabs.com - * Email: hello#infini.ltd */ - -package common - -import ( - "fmt" - "gopkg.in/yaml.v2" - "infini.sh/framework/core/util" - "testing" -) - -func TestTransformSettingsToConfig(t *testing.T) { - setting := TaskSetting{ - ClusterHealth: ClusterHealthTask{ - Enabled: true, - }, - ClusterStats: ClusterStatsTask { - Enabled: true, - }, - IndexStats: IndexStatsTask{ - Enabled: true, - }, - NodeStats: NodeStatsTask{ - Enabled: true, - NodeIDs: []string{"ddddnnnn"}, - }, - } - pipelines, err := transformSettingsToConfig(&setting, "testxxx") - if err !=nil { - t.Fatal(err) - } - buf, err := yaml.Marshal(util.MapStr{ - "pipeline": pipelines, - }) - if err !=nil { - t.Fatal(err) - } - fmt.Println(string(buf)) -} diff --git a/modules/agent/model/const.go b/modules/agent/model/const.go deleted file mode 100644 index 3cea10ea..00000000 --- a/modules/agent/model/const.go +++ /dev/null @@ -1,15 +0,0 @@ -/* Copyright © INFINI Ltd. All rights reserved. - * Web: https://infinilabs.com - * Email: hello#infini.ltd */ - -package model - -const ( - StatusOnline string = "online" - StatusOffline = "offline" -) - -const ( - KVAgentIngestConfigChanged = "agent_ingest_config_changed" - KVSyncDynamicTaskSettings = "agent_sync_dynamic_task_settings" -) \ No newline at end of file diff --git a/modules/agent/state/state.go b/modules/agent/state/state.go deleted file mode 100644 index a4ff8342..00000000 --- a/modules/agent/state/state.go +++ /dev/null @@ -1,352 +0,0 @@ -/* Copyright © INFINI Ltd. All rights reserved. - * Web: https://infinilabs.com - * Email: hello#infini.ltd */ - -package state - -import ( - "context" - "fmt" - "github.com/buger/jsonparser" - log "github.com/cihub/seelog" - "gopkg.in/yaml.v2" - "infini.sh/console/modules/agent/client" - "infini.sh/console/modules/agent/common" - model2 "infini.sh/console/modules/agent/model" - "infini.sh/framework/core/model" - "infini.sh/framework/core/host" - "infini.sh/framework/core/kv" - "infini.sh/framework/core/orm" - "infini.sh/framework/core/util" - "infini.sh/framework/modules/elastic" - "runtime" - "runtime/debug" - "strconv" - "sync" - "time" -) - -var stateManager IStateManager - -func GetStateManager() IStateManager { - if stateManager == nil { - panic("agent state manager not init") - } - return stateManager -} - -func RegisterStateManager(sm IStateManager) { - stateManager = sm -} - -func IsEnabled() bool { - return stateManager != nil -} - -type IStateManager interface { - GetAgent(ID string) (*model.Instance, error) - UpdateAgent(inst *model.Instance, syncToES bool) (*model.Instance, error) - GetTaskAgent(clusterID string) (*model.Instance, error) - DeleteAgent(agentID string) error - LoopState() - Stop() - GetAgentClient() client.ClientAPI -} - -type StateManager struct { - TTL time.Duration // kv ttl - KVKey string - stopC chan struct{} - stopCompleteC chan struct{} - agentClient *client.Client - agentIds map[string]string - agentMutex sync.Mutex - workerChan chan struct{} -} - -func NewStateManager(TTL time.Duration, kvKey string, agentIds map[string]string, agentClient *client.Client) *StateManager { - return &StateManager{ - TTL: TTL, - KVKey: kvKey, - stopC: make(chan struct{}), - stopCompleteC: make(chan struct{}), - agentClient: agentClient, - agentIds: agentIds, - workerChan: make(chan struct{}, runtime.NumCPU()), - } -} - -func (sm *StateManager) checkAgentStatus() { - onlineAgentIDs, err := common.GetLatestOnlineAgentIDs(nil, int(sm.TTL.Seconds())) - if err != nil { - log.Error(err) - return - } - //add new agent to state - sm.agentMutex.Lock() - for agentID := range onlineAgentIDs { - if _, ok := sm.agentIds[agentID]; !ok { - log.Infof("status of agent [%s] changed to online", agentID) - sm.agentIds[agentID] = model2.StatusOnline - } - } - sm.agentMutex.Unlock() - for agentID, status := range sm.agentIds { - sm.workerChan <- struct{}{} - go func(agentID string) { - defer func() { - if err := recover(); err != nil { - log.Errorf("check agent [%s] status recover form panic error: %v", agentID, err) - debug.PrintStack() - } - <-sm.workerChan - }() - sm.syncSettings(agentID) - sm.syncIngestSettings(agentID) - if _, ok := onlineAgentIDs[agentID]; ok { - host.UpdateHostAgentStatus(agentID, model2.StatusOnline) - if status == model2.StatusOnline { - return - } - // status change to online - sm.agentMutex.Lock() - sm.agentIds[agentID] = model2.StatusOnline - sm.agentMutex.Unlock() - log.Infof("status of agent [%s] changed to online", agentID) - return - }else{ - // already offline - if status == model2.StatusOffline { - return - } - } - // status change to offline - sm.agentMutex.Lock() - sm.agentIds[agentID] = model2.StatusOffline - sm.agentMutex.Unlock() - ag, err := sm.GetAgent(agentID) - if err != nil { - if err != elastic.ErrNotFound { - log.Error(err) - } - return - } - ag.Status = model2.StatusOffline - log.Infof("agent [%s] is offline", ag.Endpoint) - _, err = sm.UpdateAgent(ag, true) - if err != nil { - log.Error(err) - return - } - //update host agent status - host.UpdateHostAgentStatus(ag.ID, model2.StatusOffline) - }(agentID) - - } -} -func (sm *StateManager) getLastSyncSettingsTimestamp(agentID string) int64{ - vbytes, err := kv.GetValue(model2.KVSyncDynamicTaskSettings, []byte(agentID)) - if err != nil { - log.Error(err) - } - if vbytes == nil { - return 0 - } - t, err := strconv.ParseInt(string(vbytes), 10, 64) - if err != nil { - log.Error(err) - } - - return t -} - -func (sm *StateManager) syncSettings(agentID string) { - ag, err := sm.GetAgent(agentID) - if err != nil { - if err != elastic.ErrNotFound { - log.Errorf("get agent error: %v", err) - } - return - } - newTimestamp := time.Now().UnixMilli() - lastSyncTimestamp := sm.getLastSyncSettingsTimestamp(agentID) - settings, err := common.GetAgentSettings(agentID, lastSyncTimestamp) - if err != nil { - log.Errorf("query agent settings error: %v", err) - return - } - if len(settings) == 0 { - log.Debugf("got no settings of agent [%s]", agentID) - return - } - parseResult, err := common.ParseAgentSettings(settings) - if err != nil { - log.Errorf("parse agent settings error: %v", err) - return - } - agClient := sm.GetAgentClient() - var clusterCfgs []util.MapStr - if len(parseResult.ClusterConfigs) > 0 { - for _, cfg := range parseResult.ClusterConfigs { - clusterCfg := util.MapStr{ - "name": cfg.ID, - "enabled": true, - "endpoint": cfg.GetAnyEndpoint(), - } - if cfg.BasicAuth != nil && cfg.BasicAuth.Password != ""{ - cid := cfg.ID - if cfg.ClusterUUID != "" { - cid = cfg.ClusterUUID - } - err = agClient.SetKeystoreValue(context.Background(), ag.GetEndpoint(), fmt.Sprintf("%s_password", cid), cfg.BasicAuth.Password) - if err != nil { - log.Errorf("set keystore value error: %v", err) - continue - } - clusterCfg["basic_auth"] = util.MapStr{ - "username": cfg.BasicAuth.Username, - "password": fmt.Sprintf("$[[keystore.%s_password]]", cid), - } - } - clusterCfgs = append(clusterCfgs, clusterCfg) - } - } - var dynamicCfg = util.MapStr{} - if len(clusterCfgs) > 0 { - dynamicCfg["elasticsearch"] = clusterCfgs - } - if len(parseResult.Pipelines) > 0 { - dynamicCfg["pipeline"] = parseResult.Pipelines - } - cfgBytes, err := yaml.Marshal(dynamicCfg) - if err != nil { - log.Error("serialize config to yaml error: ", err) - return - } - //TODO - err = agClient.SaveDynamicConfig(context.Background(), ag.GetEndpoint(), "dynamic_task.yml", string(cfgBytes)) - - newTimestampStr := strconv.FormatInt(newTimestamp, 10) - err = kv.AddValue(model2.KVSyncDynamicTaskSettings, []byte(agentID), []byte(newTimestampStr)) - if err != nil { - log.Error(err) - } -} -func (sm *StateManager) syncIngestSettings(agentID string) { - v, err := kv.GetValue(model2.KVAgentIngestConfigChanged, []byte(agentID)) - if err != nil { - log.Error(err) - } - if string(v) != "1" { - return - } - ag, err := sm.GetAgent(agentID) - if err != nil { - if err != elastic.ErrNotFound { - log.Errorf("get agent error: %v", err) - } - return - } - err = sm.agentClient.SaveIngestConfig(context.Background(), ag.GetEndpoint()) - if err == nil { - kv.AddValue(model2.KVAgentIngestConfigChanged,[]byte(agentID), []byte("0")) - } -} - -func (sm *StateManager) getAvailableAgent(clusterID string) (*model.Instance, error) { - agents, err := common.LoadAgentsFromES(clusterID) - if err != nil { - return nil, err - } - if len(agents) == 0 { - return nil, nil - } - for _, ag := range agents { - if ag.Status == "offline" { - continue - } - } - return nil, nil -} - -func (sm *StateManager) LoopState() { - t := time.NewTicker(30 * time.Second) - defer t.Stop() -MAINLOOP: - for { - select { - case <-sm.stopC: - sm.stopCompleteC <- struct{}{} - close(sm.workerChan) - break MAINLOOP - case <-t.C: - sm.checkAgentStatus() - } - } -} - -func (sm *StateManager) Stop() { - sm.stopC <- struct{}{} - <-sm.stopCompleteC -} - -func (sm *StateManager) GetAgent(ID string) (*model.Instance, error) { - buf, err := kv.GetValue(sm.KVKey, []byte(ID)) - if err != nil { - return nil, err - } - strTime, _ := jsonparser.GetString(buf, "timestamp") - timestamp, _ := time.Parse(time.RFC3339, strTime) - inst := &model.Instance{} - inst.ID = ID - if time.Since(timestamp) > sm.TTL { - exists, err := orm.Get(inst) - if err != nil { - return nil, err - } - if !exists { - return nil, fmt.Errorf("can not found agent [%s]", ID) - } - //inst.Timestamp = time.Now() - err = kv.AddValue(sm.KVKey, []byte(ID), util.MustToJSONBytes(inst)) - if err != nil { - log.Errorf("save agent [%s] to kv error: %v", ID, err) - } - return inst, nil - } - err = util.FromJSONBytes(buf, inst) - return inst, err -} - -func (sm *StateManager) UpdateAgent(inst *model.Instance, syncToES bool) (*model.Instance, error) { - //inst.Timestamp = time.Now() - err := kv.AddValue(sm.KVKey, []byte(inst.ID), util.MustToJSONBytes(inst)) - if syncToES { - ctx := orm.Context{ - Refresh: "wait_for", - } - err = orm.Update(&ctx, inst) - if err != nil { - return nil, err - } - } - return inst, err -} - -func (sm *StateManager) GetTaskAgent(clusterID string) (*model.Instance, error) { - return nil, nil -} - - -func (sm *StateManager) DeleteAgent(agentID string) error { - sm.agentMutex.Lock() - delete(sm.agentIds, agentID) - sm.agentMutex.Unlock() - log.Infof("delete agent [%s] from state", agentID) - - return kv.DeleteKey(sm.KVKey, []byte(agentID)) -} - -func (sm *StateManager) GetAgentClient() client.ClientAPI { - return sm.agentClient -} diff --git a/plugin/api/email/common/auth.go b/plugin/api/email/common/auth.go index a5c78301..7dd69dc5 100644 --- a/plugin/api/email/common/auth.go +++ b/plugin/api/email/common/auth.go @@ -7,11 +7,11 @@ package common import ( "infini.sh/console/model" "infini.sh/framework/core/credential" - "infini.sh/framework/core/elastic" + model2 "infini.sh/framework/core/model" "infini.sh/framework/core/orm" ) -func GetBasicAuth(srv *model.EmailServer) (basicAuth elastic.BasicAuth, err error) { +func GetBasicAuth(srv *model.EmailServer) (basicAuth model2.BasicAuth, err error) { if srv.Auth != nil && srv.Auth.Username != "" { basicAuth = *srv.Auth return @@ -28,7 +28,7 @@ func GetBasicAuth(srv *model.EmailServer) (basicAuth elastic.BasicAuth, err erro if err != nil { return } - if auth, ok := dv.(elastic.BasicAuth); ok { + if auth, ok := dv.(model2.BasicAuth); ok { basicAuth = auth } } diff --git a/plugin/api/license/api.go b/plugin/api/license/api.go index fc9e9eb1..e5533514 100644 --- a/plugin/api/license/api.go +++ b/plugin/api/license/api.go @@ -35,7 +35,7 @@ func (handler *LicenseAPI) RequestTrialLicense(w http.ResponseWriter, req *http. } //TODO implement config for the api endpoint - request:=util.NewPostRequest("https://api.infini.sh/_license/request_trial", util.MustToJSONBytes(v)) + request:=util.NewPostRequest("https://api.infini.cloud/_license/request_trial", util.MustToJSONBytes(v)) response,err:=util.ExecuteRequest(request) if err!=nil{ handler.WriteError(w,err.Error(),response.StatusCode) diff --git a/plugin/setup/setup.go b/plugin/setup/setup.go index b746a653..a27cad77 100644 --- a/plugin/setup/setup.go +++ b/plugin/setup/setup.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "fmt" "infini.sh/framework/core/kv" + "infini.sh/framework/core/model" "io" "net/http" uri2 "net/url" @@ -88,7 +89,7 @@ func (module *Module) Start() error { log.Error(err) return } - if basicAuth, ok := bv.(elastic.BasicAuth); ok { + if basicAuth, ok := bv.(model.BasicAuth); ok { err = keystore.SetValue("SYSTEM_CLUSTER_PASS", []byte(basicAuth.Password)) if err != nil { log.Error(err) @@ -203,7 +204,7 @@ func (module *Module) validate(w http.ResponseWriter, r *http.Request, ps httpro } cfg1 = elastic1.ORMConfig{} exist, err := env.ParseConfig("elastic.orm", &cfg1) - if exist && err != nil { + if exist && err != nil &&global.Env().SystemConfig.Configs.PanicOnConfigError{ panic(err) } @@ -272,7 +273,7 @@ func (module *Module) initTempClient(r *http.Request) (error, elastic.API, Setup Enabled: true, Reserved: true, Endpoint: request.Cluster.Endpoint, - BasicAuth: &elastic.BasicAuth{ + BasicAuth: &model.BasicAuth{ Username: request.Cluster.Username, Password: request.Cluster.Password, }, From e3c10817efe0386be27a7cb50717db53a4e9a10a Mon Sep 17 00:00:00 2001 From: medcl Date: Tue, 17 Oct 2023 11:26:53 +0800 Subject: [PATCH 05/36] refactoring enroll/revoke nodes --- modules/agent/api/elasticsearch.go | 216 ++++++++++++++++------------- modules/agent/api/host.go | 41 +++--- modules/agent/api/init.go | 5 +- modules/agent/api/tod.go | 62 ++++++--- 4 files changed, 186 insertions(+), 138 deletions(-) diff --git a/modules/agent/api/elasticsearch.go b/modules/agent/api/elasticsearch.go index dc5dfb4c..b3439e13 100644 --- a/modules/agent/api/elasticsearch.go +++ b/modules/agent/api/elasticsearch.go @@ -17,35 +17,12 @@ import ( "time" ) -//func (h *APIHandler) refreshESNodesInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { -// id := ps.MustGetParameter("instance_id") -// obj := model.Instance{} -// obj.ID = id -// exists, err := orm.Get(&obj) -// if !exists || err != nil { -// h.WriteJSON(w, util.MapStr{ -// "_id": id, -// "found": false, -// }, http.StatusNotFound) -// return -// } -// _, err = refreshNodesInfo(&obj) -// if err != nil { -// log.Error(err) -// h.WriteError(w, err.Error(), http.StatusInternalServerError) -// return -// } -// h.WriteAckOKJSON(w) -//} - -func refreshNodesInfo(inst *model.Instance) ([]*model.ESNodeInfo, error) { - oldNodesInfo, err := getNodesBindingToAgent(inst) +func refreshNodesInfo(inst *model.Instance) (*elastic.DiscoveryResult, error) { + oldNodesInfo, err := getEnrolledNodesByAgent(inst) if err != nil { return nil, fmt.Errorf("error on get binding nodes info: %w", err) } - log.Error("oldNodesInfo:",util.MustToJSON(oldNodesInfo)) - ctxTimeout, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() nodesInfo, err := GetElasticsearchNodesViaAgent(ctxTimeout, inst) @@ -54,32 +31,65 @@ func refreshNodesInfo(inst *model.Instance) ([]*model.ESNodeInfo, error) { return nil, fmt.Errorf("error on get nodes info from agent: %w", err) } - log.Error("nodesInfo:",util.MustToJSON(nodesInfo)) - - for _, node := range nodesInfo { - v,ok:=oldNodesInfo[node.NodeUUID] - if ok{ - node.ClusterID=v.ClusterID - } + for nodeID, node := range nodesInfo.Nodes { + v, ok := oldNodesInfo[nodeID] + if ok { + node.ClusterID = v.ClusterID + node.Enrolled = true + } } + ////not recognized by agent, need auth? + //for _, node := range nodesInfo.UnknownProcess{ + // for _, v := range node.ListenAddresses { + // //ask user to manual enroll this node + // //check local credentials, if it works, get node info + // } + //} + + // { + // //node was not recognized by agent, need auth? + // if node.HttpPort != "" { + // for _, v := range oldNodesInfo { + // if v.PublishAddress != "" { + // if util.UnifyLocalAddress(v.PublishAddress) == util.UnifyLocalAddress(node.PublishAddress) { + // node.ClusterID = v.ClusterID + // node.ClusterName = v.ClusterName + // node.NodeUUID = v.NodeUUID + // node.ClusterUuid = v.ClusterUUID + // node.NodeName = v.NodeName + // node.Path.Home = v.PathHome + // node.Path.Logs = v.PathLogs + // node.AgentID = inst.ID + // //TODO verify node info if the node id really match, need to fetch the credentials for agent + // //or let manager sync configs to this agent, verify the node info after receiving the configs + // //report any error along with this agent and node info + // break + // } + // } + // } + // } + // + //} + return nodesInfo, nil } //get nodes info via agent -func GetElasticsearchNodesViaAgent(ctx context.Context, instance *model.Instance) ([]*model.ESNodeInfo, error) { +func GetElasticsearchNodesViaAgent(ctx context.Context, instance *model.Instance) (*elastic.DiscoveryResult, error) { req := &util.Request{ Method: http.MethodGet, Path: "/elasticsearch/nodes/_discovery", Context: ctx, } - resBody := []*model.ESNodeInfo{} - err := doRequest(instance, req, &resBody) + + obj := elastic.DiscoveryResult{} + err := doRequest(instance, req, &obj) if err != nil { return nil, err } - return resBody, nil + return &obj, nil } func AuthESNode(ctx context.Context, agentBaseURL string, cfg elastic.ElasticsearchConfig) (*model.ESNodeInfo, error) { @@ -114,13 +124,20 @@ func getNodeByPidOrUUID(nodes map[int]*model.ESNodeInfo, pid int, uuid string, p } type BindingItem struct { - ClusterID string `json:"cluster_id"` - ClusterUUID string `json:"cluster_uuid"` - NodeUUID string `json:"node_uuid"` + ClusterName string `json:"cluster_name"` + ClusterUUID string `json:"cluster_uuid"` + NodeUUID string `json:"node_uuid"` + PublishAddress string `json:"publish_address"` + NodeName string `json:"node_name"` + PathLogs string `json:"path_logs"` + PathHome string `json:"path_home"` + + //infini system assigned id + ClusterID string `json:"cluster_id"` } //node -> binding item -func getNodesBindingToAgent(instance *model.Instance) (map[string]BindingItem, error) { +func getEnrolledNodesByAgent(instance *model.Instance) (map[string]BindingItem, error) { //get nodes settings where agent id = instance id q := orm.Query{ @@ -141,23 +158,22 @@ func getNodesBindingToAgent(instance *model.Instance) (map[string]BindingItem, e for _, row := range result.Result { v, ok := row.(map[string]interface{}) if ok { - x, ok := v["metadata"] + x, ok := v["payload"] if ok { - y, ok := x.(map[string]interface{}) + f, ok := x.(map[string]interface{}) if ok { - e, ok := y["labels"] + nodeID, ok := f["node_uuid"].(string) if ok { - f, ok := e.(map[string]interface{}) - if ok { - nodeID, ok := f["node_uuid"].(string) - if ok { - item := BindingItem{} - item.ClusterID = f["cluster_id"].(string) - item.ClusterUUID = f["cluster_uuid"].(string) - item.NodeUUID = nodeID - ids[item.NodeUUID] = item - } - } + item := BindingItem{} + item.ClusterID = util.ToString(f["cluster_id"]) + item.ClusterName = util.ToString(f["cluster_name"]) + item.ClusterUUID = util.ToString(f["cluster_uuid"]) + item.PublishAddress = util.ToString(f["publish_address"]) + item.NodeName = util.ToString(f["node_name"]) + item.PathHome = util.ToString(f["path_home"]) + item.PathLogs = util.ToString(f["path_logs"]) + item.NodeUUID = nodeID + ids[item.NodeUUID] = item } } } @@ -248,7 +264,7 @@ func GetElasticLogFileContent(ctx context.Context, instance *model.Instance, bod func (h *APIHandler) getLogFilesByNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { nodeID := ps.MustGetParameter("node_id") - inst, node, err := getAgentByNodeID(nodeID) + inst, pathLogs, err := getAgentByNodeID(nodeID) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) log.Error(err) @@ -262,7 +278,7 @@ func (h *APIHandler) getLogFilesByNode(w http.ResponseWriter, req *http.Request, }, http.StatusOK) return } - logFiles, err := GetElasticLogFiles(nil, inst, node.Path.Logs) + logFiles, err := GetElasticLogFiles(nil, inst, pathLogs) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) log.Error(err) @@ -276,7 +292,7 @@ func (h *APIHandler) getLogFilesByNode(w http.ResponseWriter, req *http.Request, func (h *APIHandler) getLogFileContent(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { nodeID := ps.MustGetParameter("node_id") - inst, node, err := getAgentByNodeID(nodeID) + inst, pathLogs, err := getAgentByNodeID(nodeID) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) log.Error(err) @@ -299,7 +315,7 @@ func (h *APIHandler) getLogFileContent(w http.ResponseWriter, req *http.Request, log.Error(err) return } - reqBody.LogsPath = node.Path.Logs + reqBody.LogsPath = pathLogs res, err := GetElasticLogFileContent(nil, inst, reqBody) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) @@ -309,45 +325,57 @@ func (h *APIHandler) getLogFileContent(w http.ResponseWriter, req *http.Request, h.WriteJSON(w, res, http.StatusOK) } -func getAgentByNodeID(nodeID string) (*model.Instance, *model.ESNodeInfo, error) { - queryDsl := util.MapStr{ - "size": 1, - "query": util.MapStr{ - "term": util.MapStr{ - "node_uuid": util.MapStr{ - "value": nodeID, - }, - }, - }, - "sort": []util.MapStr{ - { - "timestamp": util.MapStr{ - "order": "desc", - }, - }, - }, +//instance, pathLogs +func getAgentByNodeID(nodeID string) (*model.Instance, string, error) { + + q := orm.Query{ + Size: 1000, + Conds: orm.And(orm.Eq("metadata.category", "node_settings"), + orm.Eq("metadata.name", "agent"), + orm.Eq("payload.node_uuid", nodeID), + ), } - q := &orm.Query{ - RawQuery: util.MustToJSONBytes(queryDsl), - } - err, result := orm.Search(model.ESNodeInfo{}, q) + + err, result := orm.Search(model.Setting{}, &q) if err != nil { - return nil, nil, err + return nil, "", err } - if len(result.Result) > 0 { - buf := util.MustToJSONBytes(result.Result[0]) - v := &model.ESNodeInfo{} - err = util.FromJSONBytes(buf, v) - inst := &model.Instance{} - inst.ID = v.AgentID - _, err = orm.Get(inst) - if err != nil { - return nil, v, err + + for _, row := range result.Result { + v, ok := row.(map[string]interface{}) + if ok { + pathLogs := "" + payload, ok := v["payload"] + if ok { + payloadMap, ok := payload.(map[string]interface{}) + if ok { + pathLogs = util.ToString(payloadMap["path_logs"]) + } + } + + x, ok := v["metadata"] + if ok { + f, ok := x.(map[string]interface{}) + if ok { + labels, ok := f["labels"].(map[string]interface{}) + if ok { + id, ok := labels["agent_id"] + if ok { + inst := &model.Instance{} + inst.ID = util.ToString(id) + _, err = orm.Get(inst) + if err != nil { + return nil, pathLogs, err + } + if inst.Name == "" { + return nil, pathLogs, nil + } + return inst, pathLogs, nil + } + } + } + } } - if inst.Name == "" { - return nil, v, nil - } - return inst, v, nil } - return nil, nil, nil + return nil, "", nil } diff --git a/modules/agent/api/host.go b/modules/agent/api/host.go index 16419022..80f8b197 100644 --- a/modules/agent/api/host.go +++ b/modules/agent/api/host.go @@ -5,7 +5,6 @@ package api import ( - "context" "fmt" log "github.com/cihub/seelog" httprouter "infini.sh/framework/core/api/router" @@ -197,26 +196,26 @@ func (h *APIHandler) GetHostElasticProcess(w http.ResponseWriter, req *http.Requ return } - esNodesInfo, err := GetElasticsearchNodesViaAgent(context.Background(), &obj) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - var processes []util.MapStr - for _, node := range esNodesInfo { - processes = append(processes, util.MapStr{ - "pid": node.ProcessInfo.PID, - "pid_status": node.ProcessInfo.Status, - "cluster_name": node.ClusterName, - "cluster_uuid": node.ClusterUuid, - "cluster_id": node.ClusterID, - "node_id": node.NodeUUID, - "node_name": node.NodeName, - "uptime_in_ms": time.Now().UnixMilli() - node.ProcessInfo.CreateTime, - }) - } + //esNodesInfo, err := GetElasticsearchNodesViaAgent(context.Background(), &obj) + //if err != nil { + // log.Error(err) + // h.WriteError(w, err.Error(), http.StatusInternalServerError) + // return + //} + //var processes []util.MapStr + //for _, node := range esNodesInfo { + // processes = append(processes, util.MapStr{ + // "pid": node.ProcessInfo.PID, + // "pid_status": node.ProcessInfo.Status, + // "cluster_name": node.ClusterName, + // "cluster_uuid": node.ClusterUuid, + // "cluster_id": node.ClusterID, + // "node_id": node.NodeUUID, + // "node_name": node.NodeName, + // "uptime_in_ms": time.Now().UnixMilli() - node.ProcessInfo.CreateTime, + // }) + //} h.WriteJSON(w, util.MapStr{ - "elastic_processes": processes, + //"elastic_processes": processes, }, http.StatusOK) } \ No newline at end of file diff --git a/modules/agent/api/init.go b/modules/agent/api/init.go index 209c514a..166da6e6 100644 --- a/modules/agent/api/init.go +++ b/modules/agent/api/init.go @@ -16,7 +16,10 @@ func Init() { api.HandleAPIMethod(api.GET, "/host/:host_id/processes", handler.GetHostElasticProcess) api.HandleAPIMethod(api.DELETE, "/host/:host_id", handler.deleteHost) - api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_associate", handler.RequirePermission(handler.associateESNode, enum.PermissionAgentInstanceWrite)) + //bind agent with nodes + api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_enroll", handler.RequirePermission(handler.enrollESNode, enum.PermissionAgentInstanceWrite)) + api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_revoke", handler.RequirePermission(handler.revokeESNode, enum.PermissionAgentInstanceWrite)) + api.HandleAPIMethod(api.POST, "/auto_associate", handler.RequirePermission(handler.autoAssociateESNode, enum.PermissionAgentInstanceWrite)) //api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_auth", handler.RequirePermission(handler.authESNode, enum.PermissionAgentInstanceWrite)) diff --git a/modules/agent/api/tod.go b/modules/agent/api/tod.go index 72827ef4..38811fef 100644 --- a/modules/agent/api/tod.go +++ b/modules/agent/api/tod.go @@ -183,7 +183,7 @@ func (h *APIHandler) authESNode(w http.ResponseWriter, req *http.Request, ps htt h.WriteJSON(w, fmt.Sprintf("node [%s] of agent [%s] was not found", oldNodeInfo.ID, inst.Name), http.StatusInternalServerError) return } - }else{ + } else { //find out the node id with credentials cfg := reqBody.ESConfig if cfg.Endpoint == "" { @@ -222,10 +222,9 @@ func (h *APIHandler) authESNode(w http.ResponseWriter, req *http.Request, ps htt // return //} - reqBody.NodeID=nodeInfo.NodeUUID + reqBody.NodeID = nodeInfo.NodeUUID } - //nodeInfo:=elastic.NodeConfig{} //nodeInfo.ID = reqBody.NodeID //nodeInfo.AgentID = inst.ID @@ -253,7 +252,7 @@ func NewClusterSettings(clusterID string) *model.Setting { return &settings } -func NewNodeAgentSettings(clusterID, clusterUUID, nodeUUID, agentID, agentCredential string) *model.Setting { +func NewNodeAgentSettings(instanceID string, item *BindingItem) *model.Setting { settings := model.Setting{ Metadata: model.SettingsMetadata{ @@ -261,14 +260,21 @@ func NewNodeAgentSettings(clusterID, clusterUUID, nodeUUID, agentID, agentCreden Name: "agent", }, } - settings.ID = fmt.Sprintf("%v_%v_%v", settings.Metadata.Category, settings.Metadata.Name, nodeUUID) + settings.ID = fmt.Sprintf("%v_%v_%v", settings.Metadata.Category, settings.Metadata.Name, item.NodeUUID) settings.Metadata.Labels = util.MapStr{ - "cluster_id": clusterID, - "cluster_uuid": clusterUUID, - "node_uuid": nodeUUID, - "agent_id": agentID, - "agent_credential": agentCredential, + "agent_id": instanceID, + } + + settings.Payload = util.MapStr{ + "cluster_id": item.ClusterID, + "cluster_name": item.ClusterName, + "cluster_uuid": item.ClusterUUID, + "node_uuid": item.NodeUUID, + "publish_address": item.PublishAddress, + "node_name": item.NodeName, + "path_home": item.PathHome, + "path_logs": item.PathLogs, } return &settings @@ -298,33 +304,45 @@ const Cluster = "cluster_settings" const Node = "node_settings" const Index = "index_settings" -func (h *APIHandler) associateESNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +func (h *APIHandler) revokeESNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + //agent id + instID := ps.MustGetParameter("instance_id") + item := BindingItem{} + err := h.DecodeJSON(req, &item) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + settings := NewNodeAgentSettings(instID, &item) + err = orm.Delete(&orm.Context{ + Refresh: "wait_for", + }, settings) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } +} + +func (h *APIHandler) enrollESNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { //agent id instID := ps.MustGetParameter("instance_id") //node id and cluster id - reqBody := struct { - ClusterUUID string `json:"cluster_uuid"` - NodeUUID string `json:"node_uuid"` - - //infini system assigned id - ClusterID string `json:"cluster_id"` - }{} - err := h.DecodeJSON(req, &reqBody) + item := BindingItem{} + err := h.DecodeJSON(req, &item) if err != nil { - log.Error(err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } //update node's setting - settings := NewNodeAgentSettings(reqBody.ClusterID, reqBody.ClusterUUID, reqBody.NodeUUID, instID, "node.AgentCredential") + settings := NewNodeAgentSettings(instID, &item) err = orm.Update(&orm.Context{ Refresh: "wait_for", }, settings) if err != nil { - log.Error(err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } From 4f949cfd4005aaf1ed22a0d327f51bac4374e3ca Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 19 Oct 2023 10:56:31 +0800 Subject: [PATCH 06/36] update api --- modules/agent/api/init.go | 4 ++-- modules/agent/api/tod.go | 18 ------------------ 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/modules/agent/api/init.go b/modules/agent/api/init.go index 166da6e6..9dc23296 100644 --- a/modules/agent/api/init.go +++ b/modules/agent/api/init.go @@ -17,14 +17,14 @@ func Init() { api.HandleAPIMethod(api.DELETE, "/host/:host_id", handler.deleteHost) //bind agent with nodes + api.HandleAPIMethod(api.GET, "/instance/:instance_id/node/_discovery", handler.RequirePermission(handler.getESNodesInfo, enum.PermissionAgentInstanceRead)) api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_enroll", handler.RequirePermission(handler.enrollESNode, enum.PermissionAgentInstanceWrite)) api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_revoke", handler.RequirePermission(handler.revokeESNode, enum.PermissionAgentInstanceWrite)) api.HandleAPIMethod(api.POST, "/auto_associate", handler.RequirePermission(handler.autoAssociateESNode, enum.PermissionAgentInstanceWrite)) //api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_auth", handler.RequirePermission(handler.authESNode, enum.PermissionAgentInstanceWrite)) - api.HandleAPIMethod(api.DELETE, "/instance/:instance_id/_nodes", handler.RequirePermission(handler.deleteESNode, enum.PermissionAgentInstanceWrite)) - api.HandleAPIMethod(api.GET, "/instance/:instance_id/_nodes", handler.RequirePermission(handler.getESNodesInfo, enum.PermissionAgentInstanceRead)) + //api.HandleAPIMethod(api.DELETE, "/instance/:instance_id/_nodes", handler.RequirePermission(handler.deleteESNode, enum.PermissionAgentInstanceWrite)) //get elasticsearch node logs, direct fetch or via stored logs(TODO) api.HandleAPIMethod(api.GET, "/elasticsearch/:id/node/:node_id/logs/_list", handler.RequirePermission(handler.getLogFilesByNode, enum.PermissionAgentInstanceRead)) diff --git a/modules/agent/api/tod.go b/modules/agent/api/tod.go index 38811fef..a101b391 100644 --- a/modules/agent/api/tod.go +++ b/modules/agent/api/tod.go @@ -347,24 +347,6 @@ func (h *APIHandler) enrollESNode(w http.ResponseWriter, req *http.Request, ps h return } - //settings, err := common2.GetAgentSettings(instID, 0) - //if err != nil { - // log.Error(err) - // h.WriteError(w, err.Error(), http.StatusInternalServerError) - // return - //} - //setting := pickAgentSettings(settings, node) - //if setting == nil { - // setting, err = getAgentTaskSetting(instID, node) - // if err != nil { - // log.Error("get agent task setting error: ", err) - // } - // err = orm.Create(nil, setting) - // if err != nil { - // log.Error("save agent task setting error: ", err) - // } - //} - h.WriteAckOKJSON(w) } From f6fe743a888983a879727c05dd1fc3deea541994 Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 19 Oct 2023 14:18:48 +0800 Subject: [PATCH 07/36] update enroll staus --- modules/agent/api/elasticsearch.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/agent/api/elasticsearch.go b/modules/agent/api/elasticsearch.go index b3439e13..c03986ae 100644 --- a/modules/agent/api/elasticsearch.go +++ b/modules/agent/api/elasticsearch.go @@ -18,7 +18,7 @@ import ( ) func refreshNodesInfo(inst *model.Instance) (*elastic.DiscoveryResult, error) { - oldNodesInfo, err := getEnrolledNodesByAgent(inst) + enrolledNodesByAgent, err := getEnrolledNodesByAgent(inst) if err != nil { return nil, fmt.Errorf("error on get binding nodes info: %w", err) } @@ -30,9 +30,9 @@ func refreshNodesInfo(inst *model.Instance) (*elastic.DiscoveryResult, error) { //TODO return already biding nodes info ?? return nil, fmt.Errorf("error on get nodes info from agent: %w", err) } - for nodeID, node := range nodesInfo.Nodes { - v, ok := oldNodesInfo[nodeID] + + v, ok := enrolledNodesByAgent[nodeID] if ok { node.ClusterID = v.ClusterID node.Enrolled = true @@ -50,7 +50,7 @@ func refreshNodesInfo(inst *model.Instance) (*elastic.DiscoveryResult, error) { // { // //node was not recognized by agent, need auth? // if node.HttpPort != "" { - // for _, v := range oldNodesInfo { + // for _, v := range enrolledNodesByAgent { // if v.PublishAddress != "" { // if util.UnifyLocalAddress(v.PublishAddress) == util.UnifyLocalAddress(node.PublishAddress) { // node.ClusterID = v.ClusterID From ddba189aec34195b02461a260e201cfad39a2a8a Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 19 Oct 2023 14:30:33 +0800 Subject: [PATCH 08/36] ack after revoke --- modules/agent/api/tod.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/agent/api/tod.go b/modules/agent/api/tod.go index a101b391..0a457bbc 100644 --- a/modules/agent/api/tod.go +++ b/modules/agent/api/tod.go @@ -322,6 +322,7 @@ func (h *APIHandler) revokeESNode(w http.ResponseWriter, req *http.Request, ps h h.WriteError(w, err.Error(), http.StatusInternalServerError) return } + h.WriteAckOKJSON(w) } func (h *APIHandler) enrollESNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { From 9bab385d8cd32a7fb0682ce986f25448bfb12e2b Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 19 Oct 2023 21:14:49 +0800 Subject: [PATCH 09/36] support remote config store --- config/ingest_config.tpl | 90 ------- config/task_config.tpl | 84 ------- config_repo/configs/cluster_xx_node_xx.yml | 17 -- config_repo/settings.yml | 21 +- config_repo/templates/task_config.tpl | 5 +- main.go | 2 +- modules/agent/api/elasticsearch.go | 262 +++++++++++++++++---- modules/agent/api/init.go | 5 + modules/agent/api/tod.go | 10 +- 9 files changed, 232 insertions(+), 264 deletions(-) delete mode 100644 config/ingest_config.tpl delete mode 100644 config/task_config.tpl delete mode 100644 config_repo/configs/cluster_xx_node_xx.yml diff --git a/config/ingest_config.tpl b/config/ingest_config.tpl deleted file mode 100644 index d93dffa9..00000000 --- a/config/ingest_config.tpl +++ /dev/null @@ -1,90 +0,0 @@ -elasticsearch: - - name: $[[INGEST_CLUSTER_ID]] - enabled: true - endpoints: $[[INGEST_CLUSTER_ENDPOINT]] - discovery: - enabled: false - basic_auth: - username: $[[INGEST_CLUSTER_USERNAME]] - password: $[[keystore.ingest_cluster_password]] - -metrics: - enabled: true - queue: metrics - network: - enabled: true - summary: true - details: true - memory: - metrics: - - swap - - memory - disk: - metrics: - - iops - - usage - cpu: - metrics: - - idle - - system - - user - - iowait - - load - instance: - enabled: true - -elastic: - availability_check: - enabled: false - -pipeline: - - name: merge_logs - auto_start: true - keep_running: true - processor: - - indexing_merge: - index_name: ".infini_logs" - elasticsearch: "$[[INGEST_CLUSTER_ID]]" - input_queue: "logs" - idle_timeout_in_seconds: 10 - output_queue: - name: "merged_requests" - worker_size: 1 - bulk_size_in_mb: 5 - - name: merge_metrics - auto_start: true - keep_running: true - processor: - - indexing_merge: - elasticsearch: "$[[INGEST_CLUSTER_ID]]" - index_name: ".infini_metrics" - input_queue: "metrics" - output_queue: - name: "merged_requests" - worker_size: 1 - bulk_size_in_mb: 5 - - name: ingest_merged_requests - auto_start: true - keep_running: true - processor: - - bulk_indexing: - max_worker_size: 1 - bulk: - batch_size_in_mb: 5 - batch_size_in_docs: 5000 - max_retry_times: 0 - invalid_queue: "" - response_handle: - include_index_stats: false - include_action_stats: false - output_bulk_stats: false - include_error_details: false - save_error_results: false - save_success_results: false - save_busy_results: false - consumer: - fetch_max_messages: 5 - queues: - type: indexing_merge - when: - cluster_available: ["$[[INGEST_CLUSTER_ID]]"] \ No newline at end of file diff --git a/config/task_config.tpl b/config/task_config.tpl deleted file mode 100644 index c5abc715..00000000 --- a/config/task_config.tpl +++ /dev/null @@ -1,84 +0,0 @@ -elasticsearch: - - id: $[[CLUSTER_ID]] - name: $[[CLUSTER_ID]] - enabled: true - endpoint: $[[CLUSTER_ENDPOINT]] - discovery: - enabled: false - basic_auth: - username: $[[CLUSTER_USERNAME]] - password: $[[keystore.$[[CLUSTER_ID]]_password]] - -pipeline: -#clsuter level metrics -- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] - enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] - keep_running: true - singleton: true - name: collect_$[[CLUSTER_ID]]_es_cluster_stats - retry_delay_in_ms: 10000 - processor: - - es_cluster_stats: - elasticsearch: $[[CLUSTER_ID]] - labels: - cluster_id: $[[CLUSTER_ID]] - when: - cluster_available: ["$[[CLUSTER_ID]]"] - -- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] - enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] - keep_running: true - singleton: true - name: collect_$[[CLUSTER_ID]]_es_index_stats - retry_delay_in_ms: 10000 - processor: - - es_index_stats: - elasticsearch: $[[CLUSTER_ID]] - labels: - cluster_id: $[[CLUSTER_ID]] - when: - cluster_available: ["$[[CLUSTER_ID]]"] - -- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] - enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] - keep_running: true - singleton: true - name: collect_$[[CLUSTER_ID]]_es_cluster_health - retry_delay_in_ms: 10000 - processor: - - es_cluster_health: - elasticsearch: $[[CLUSTER_ID]] - labels: - cluster_id: $[[CLUSTER_ID]] - when: - cluster_available: ["$[[CLUSTER_ID]]"] - -#node level metrics -- auto_start: $[[NODE_LEVEL_TASKS_ENABLED]] - enabled: $[[NODE_LEVEL_TASKS_ENABLED]] - keep_running: true - name: collect_$[[CLUSTER_ID]]_es_node_stats - retry_delay_in_ms: 10000 - processor: - - es_node_stats: - elasticsearch: $[[CLUSTER_ID]] - labels: - cluster_id: $[[CLUSTER_ID]] - when: - cluster_available: ["$[[CLUSTER_ID]]"] - -#node logs -- auto_start: $[[NODE_LEVEL_TASKS_ENABLED]] - enabled: $[[NODE_LEVEL_TASKS_ENABLED]] - keep_running: true - name: collect_$[[CLUSTER_ID]]_es_logs - retry_delay_in_ms: 10000 - processor: - - es_logs_processor: - elasticsearch: $[[CLUSTER_ID]] - labels: - cluster_id: $[[CLUSTER_ID]] - logs_path: $[[NODE_LOGS_PATH]] - queue_name: logs - when: - cluster_available: ["$[[CLUSTER_ID]]"] diff --git a/config_repo/configs/cluster_xx_node_xx.yml b/config_repo/configs/cluster_xx_node_xx.yml deleted file mode 100644 index d67e7cc6..00000000 --- a/config_repo/configs/cluster_xx_node_xx.yml +++ /dev/null @@ -1,17 +0,0 @@ -configs.template: - - name: "cluster_a_node_b_task_config" - path: ./config/task_config.tpl - variable: - CLUSTER_ID: infini_default_ingest_cluster - CLUSTER_ENDPOINT: ["https://localhost:9200"] - CLUSTER_USERNAME: "admin" - CLUSTER_VER: "1.6.0" - CLUSTER_DISTRIBUTION: "easysearch" - INDEX_PREFIX: ".infini_" - CLUSTER_LEVEL_TASKS_ENABLED: false - NODE_LEVEL_TASKS_ENABLED: true - NODE_LOGS_PATH: "/opt/easysearch/logs/" - - -#MANAGED_CONFIG_VERSION: 19 -#MANAGED: true \ No newline at end of file diff --git a/config_repo/settings.yml b/config_repo/settings.yml index 41d42024..722d8df2 100644 --- a/config_repo/settings.yml +++ b/config_repo/settings.yml @@ -4,14 +4,13 @@ configs: #define configs group - ./templates/ingest_config.tpl - ./templates/task_config.tpl - ./configs/ingest_config.yml - - ./configs/cluster_xx_node_xx.yml instances: #define which config instance should fetch - infini_default_system_cluster: #instance group - plugins: - - ingest - instances: - - ck0mkk805f5virpsejp0 - - ckjrpdg05f5lrfp8qlng + _all: #instance group +# plugins: +# - ingest +# instances: +# - ck0mkk805f5virpsejp0 +# - ckjrpdg05f5lrfp8qlng configs: - general_ingest_template secrets: @@ -22,10 +21,4 @@ secrets: keystore: ingest_cluster_password: type: plaintext - value: "d7cc48e69a41dac719fb" - infini_default_ingest_cluster_password: - type: plaintext - value: "d7cc48e69a41dac719fb" -# infini_default_ingest_cluster_password: -# type: credential -# value: "ckghspo05f5q7pr20ct0" #credential_id \ No newline at end of file + value: "d7cc48e69a41dac719fb" \ No newline at end of file diff --git a/config_repo/templates/task_config.tpl b/config_repo/templates/task_config.tpl index 119d2fc8..7d30522e 100644 --- a/config_repo/templates/task_config.tpl +++ b/config_repo/templates/task_config.tpl @@ -1,3 +1,6 @@ +env: + CLUSTER_PASSWORD: $[[keystore.$[[CLUSTER_ID]]_password]] + elasticsearch: - id: $[[CLUSTER_ID]] name: $[[CLUSTER_ID]] @@ -7,7 +10,7 @@ elasticsearch: enabled: false basic_auth: username: $[[CLUSTER_USERNAME]] - password: $[[keystore.$[[CLUSTER_ID]]_password]] + password: $[[CLUSTER_PASSWORD]] pipeline: #clsuter level metrics diff --git a/main.go b/main.go index 095ffbba..63e76815 100644 --- a/main.go +++ b/main.go @@ -129,7 +129,6 @@ func main() { orm.RegisterSchemaWithIndexName(elastic.View{}, "view") orm.RegisterSchemaWithIndexName(elastic.CommonCommand{}, "commands") //orm.RegisterSchema(elastic.TraceTemplate{}, "trace-template") - //orm.RegisterSchema(model.Instance{}, "instance") orm.RegisterSchemaWithIndexName(alerting.Rule{}, "alert-rule") orm.RegisterSchemaWithIndexName(alerting.Alert{}, "alert-history") orm.RegisterSchemaWithIndexName(alerting.AlertMessage{}, "alert-message") @@ -142,6 +141,7 @@ func main() { orm.RegisterSchemaWithIndexName(model.Notification{}, "notification") orm.RegisterSchemaWithIndexName(model.EmailServer{}, "email-server") orm.RegisterSchemaWithIndexName(model2.Instance{}, "instance") + orm.RegisterSchemaWithIndexName(api3.RemoteConfig{}, "configs") api.RegisterSchema() diff --git a/modules/agent/api/elasticsearch.go b/modules/agent/api/elasticsearch.go index c03986ae..2b54c739 100644 --- a/modules/agent/api/elasticsearch.go +++ b/modules/agent/api/elasticsearch.go @@ -5,6 +5,7 @@ package api import ( + "bytes" "context" "fmt" log "github.com/cihub/seelog" @@ -13,12 +14,70 @@ import ( "infini.sh/framework/core/model" "infini.sh/framework/core/orm" "infini.sh/framework/core/util" + common2 "infini.sh/framework/modules/elastic/common" + "infini.sh/framework/plugins/managed/common" "net/http" "time" ) +//node -> binding item +func GetEnrolledNodesByAgent(instance *model.Instance) (map[string]BindingItem, error) { + + //get nodes settings where agent id = instance id + q := orm.Query{ + Size: 1000, + Conds: orm.And(orm.Eq("metadata.category", "node_settings"), + orm.Eq("metadata.name", "agent"), + orm.Eq("metadata.labels.agent_id", instance.ID), + ), + } + + err, result := orm.Search(model.Setting{}, &q) + + if err != nil { + return nil, err + } + + ids := map[string]BindingItem{} + for _, row := range result.Result { + v, ok := row.(map[string]interface{}) + if ok { + x, ok := v["payload"] + if ok { + f, ok := x.(map[string]interface{}) + if ok { + nodeID, ok := f["node_uuid"].(string) + if ok { + item := BindingItem{} + item.ClusterID = util.ToString(f["cluster_id"]) + item.ClusterName = util.ToString(f["cluster_name"]) + item.ClusterUUID = util.ToString(f["cluster_uuid"]) + item.PublishAddress = util.ToString(f["publish_address"]) + item.NodeName = util.ToString(f["node_name"]) + item.PathHome = util.ToString(f["path_home"]) + item.PathLogs = util.ToString(f["path_logs"]) + item.NodeUUID = nodeID + + t, ok := v["updated"] + if ok { + layout := "2006-01-02T15:04:05.999999-07:00" + t1, err := time.Parse(layout, util.ToString(t)) + if err == nil { + item.Updated = t1.Unix() + } + } + ids[item.NodeUUID] = item + + } + } + } + } + } + return ids, nil +} + func refreshNodesInfo(inst *model.Instance) (*elastic.DiscoveryResult, error) { - enrolledNodesByAgent, err := getEnrolledNodesByAgent(inst) + enrolledNodesByAgent, err := GetEnrolledNodesByAgent(inst) if err != nil { return nil, fmt.Errorf("error on get binding nodes info: %w", err) } @@ -30,13 +89,14 @@ func refreshNodesInfo(inst *model.Instance) (*elastic.DiscoveryResult, error) { //TODO return already biding nodes info ?? return nil, fmt.Errorf("error on get nodes info from agent: %w", err) } + for nodeID, node := range nodesInfo.Nodes { - v, ok := enrolledNodesByAgent[nodeID] - if ok { - node.ClusterID = v.ClusterID - node.Enrolled = true - } + v, ok := enrolledNodesByAgent[nodeID] + if ok { + node.ClusterID = v.ClusterID + node.Enrolled = true + } } ////not recognized by agent, need auth? @@ -75,6 +135,149 @@ func refreshNodesInfo(inst *model.Instance) (*elastic.DiscoveryResult, error) { return nodesInfo, nil } +type RemoteConfig struct { + orm.ORMObjectBase + Metadata model.Metadata `json:"metadata" elastic_mapping:"metadata: { type: object }"` + Payload common.ConfigFile `json:"payload" elastic_mapping:"payload: { type: object}"` +} + +func remoteConfigProvider(instance model.Instance) []*common.ConfigFile { + + //fetch configs from remote db + //fetch configs assigned to (instance=_all OR instance=$instance_id ) AND application.name=$application.name + + q := orm.Query{ + Size: 1000, + Conds: orm.And(orm.Eq("metadata.category", "app_settings"), + orm.Eq("metadata.name", instance.Application.Name), + orm.Eq("metadata.labels.instance", "_all"), + ), + } + + err, searchResult := orm.Search(RemoteConfig{}, &q) + if err != nil { + panic(err) + } + + result := []*common.ConfigFile{} + + for _, row := range searchResult.Result { + v, ok := row.(map[string]interface{}) + if ok { + x, ok := v["payload"] + if ok { + f, ok := x.(map[string]interface{}) + if ok { + name, ok := f["name"].(string) + if ok { + item := common.ConfigFile{} + item.Name = util.ToString(name) + item.Location = util.ToString(f["location"]) + item.Content = util.ToString(f["content"]) + item.Version,_ = util.ToInt64(util.ToString(f["version"])) + item.Size = int64(len(item.Content)) + item.Managed = true + t, ok := v["updated"] + if ok { + layout := "2006-01-02T15:04:05.999999-07:00" + t1, err := time.Parse(layout, util.ToString(t)) + if err == nil { + item.Updated = t1.Unix() + } + } + result=append(result,&item) + } + } + } + } + } + + log.Error("remoteConfigProvider", "result", result) + + return result +} + +func dynamicAgentConfigProvider(instance model.Instance) []*common.ConfigFile { + + //get config files from remote db + //get settings with this agent id + result := []*common.ConfigFile{} + ids, err := GetEnrolledNodesByAgent(&instance) + if err != nil { + panic(err) + } + + var latestTimestamp int64 + for _, v := range ids { + if v.Updated > latestTimestamp { + latestTimestamp = v.Updated + } + } + + if len(ids) > 0 { + + cfg := common.ConfigFile{} + cfg.Name = "generated_metrics_tasks.yml" + cfg.Location = "generated_metrics_tasks.yml" + cfg.Content = getConfigs(ids) + cfg.Size = int64(len(cfg.Content)) + cfg.Version = latestTimestamp + cfg.Managed = true + cfg.Updated = latestTimestamp + result = append(result, &cfg) + } + + return result +} + +func getConfigs(items map[string]BindingItem) string { + buffer := bytes.NewBuffer([]byte("configs.template:\n ")) + + for _, v := range items { + + if v.ClusterID == "" { + panic("cluster id is empty") + } + metadata := elastic.GetMetadata(v.ClusterID) + var clusterLevelEnabled = false + var nodeLevelEnabled = true + var clusterEndPoint = metadata.Config.GetAnyEndpoint() + credential, err := common2.GetCredential(metadata.Config.CredentialID) + if err != nil { + panic(err) + } + var dv interface{} + dv, err = credential.Decode() + if err != nil { + panic(err) + } + var username = "" + var password = "" + + if auth, ok := dv.(model.BasicAuth); ok { + username = auth.Username + password = auth.Password + } + + buffer.Write([]byte(fmt.Sprintf("- name: \"%v\"\n path: ./config/task_config.tpl\n "+ + "variable:\n "+ + "CLUSTER_ID: %v\n "+ + "CLUSTER_ENDPOINT: [\"%v\"]\n "+ + "CLUSTER_USERNAME: \"%v\"\n "+ + "CLUSTER_PASSWORD: \"%v\"\n "+ + "CLUSTER_LEVEL_TASKS_ENABLED: %v\n "+ + "NODE_LEVEL_TASKS_ENABLED: %v\n "+ + "NODE_LOGS_PATH: \"%v\"\n\n\n"+ + "#MANAGED_CONFIG_VERSION: %v\n"+ + "#MANAGED: true", + v.NodeUUID, v.ClusterID, clusterEndPoint, username, password, clusterLevelEnabled, nodeLevelEnabled, v.PathLogs, v.Updated))) + } + + //password: $[[keystore.$[[CLUSTER_ID]]_password]] + + return buffer.String() +} + //get nodes info via agent func GetElasticsearchNodesViaAgent(ctx context.Context, instance *model.Instance) (*elastic.DiscoveryResult, error) { req := &util.Request{ @@ -134,52 +337,7 @@ type BindingItem struct { //infini system assigned id ClusterID string `json:"cluster_id"` -} - -//node -> binding item -func getEnrolledNodesByAgent(instance *model.Instance) (map[string]BindingItem, error) { - - //get nodes settings where agent id = instance id - q := orm.Query{ - Size: 1000, - Conds: orm.And(orm.Eq("metadata.category", "node_settings"), - orm.Eq("metadata.name", "agent"), - orm.Eq("metadata.labels.agent_id", instance.ID), - ), - } - - err, result := orm.Search(model.Setting{}, &q) - - if err != nil { - return nil, err - } - - ids := map[string]BindingItem{} - for _, row := range result.Result { - v, ok := row.(map[string]interface{}) - if ok { - x, ok := v["payload"] - if ok { - f, ok := x.(map[string]interface{}) - if ok { - nodeID, ok := f["node_uuid"].(string) - if ok { - item := BindingItem{} - item.ClusterID = util.ToString(f["cluster_id"]) - item.ClusterName = util.ToString(f["cluster_name"]) - item.ClusterUUID = util.ToString(f["cluster_uuid"]) - item.PublishAddress = util.ToString(f["publish_address"]) - item.NodeName = util.ToString(f["node_name"]) - item.PathHome = util.ToString(f["path_home"]) - item.PathLogs = util.ToString(f["path_logs"]) - item.NodeUUID = nodeID - ids[item.NodeUUID] = item - } - } - } - } - } - return ids, nil + Updated int64 `json:"updated"` } func getUnAssociateNodes() (map[string][]model.ESNodeInfo, error) { diff --git a/modules/agent/api/init.go b/modules/agent/api/init.go index 9dc23296..d863e023 100644 --- a/modules/agent/api/init.go +++ b/modules/agent/api/init.go @@ -7,6 +7,7 @@ package api import ( "infini.sh/framework/core/api" "infini.sh/framework/core/api/rbac/enum" + "infini.sh/framework/plugins/managed/server" ) func Init() { @@ -32,4 +33,8 @@ func Init() { //api.HandleAPIMethod(api.POST, "/agent/install_command", handler.RequireLogin(handler.generateInstallCommand)) //api.HandleAPIMethod(api.GET, "/agent/install.sh", handler.getInstallScript) + + + server.RegisterConfigProvider(remoteConfigProvider) + server.RegisterConfigProvider(dynamicAgentConfigProvider) } diff --git a/modules/agent/api/tod.go b/modules/agent/api/tod.go index 0a457bbc..da6c172f 100644 --- a/modules/agent/api/tod.go +++ b/modules/agent/api/tod.go @@ -239,7 +239,7 @@ func (h *APIHandler) authESNode(w http.ResponseWriter, req *http.Request, ps htt func NewClusterSettings(clusterID string) *model.Setting { settings := model.Setting{ - Metadata: model.SettingsMetadata{ + Metadata: model.Metadata{ Category: Cluster, }, } @@ -255,7 +255,7 @@ func NewClusterSettings(clusterID string) *model.Setting { func NewNodeAgentSettings(instanceID string, item *BindingItem) *model.Setting { settings := model.Setting{ - Metadata: model.SettingsMetadata{ + Metadata: model.Metadata{ Category: Node, Name: "agent", }, @@ -283,7 +283,7 @@ func NewNodeAgentSettings(instanceID string, item *BindingItem) *model.Setting { func NewIndexSettings(clusterID, nodeID, agentID, indexName, indexID string) *model.Setting { settings := model.Setting{ - Metadata: model.SettingsMetadata{ + Metadata: model.Metadata{ Category: Index, }, } @@ -469,7 +469,7 @@ func (h *APIHandler) autoAssociateESNode(w http.ResponseWriter, req *http.Reques taskSetting.ClusterStats = nil } setting = &model.Setting{ - Metadata: model.SettingsMetadata{ + Metadata: model.Metadata{ Category: "agent", Name: "task", Labels: util.MapStr{ @@ -623,7 +623,7 @@ func getAgentTaskSetting(agentID string, v model.ESNodeInfo) (*model.Setting, er LogsPath: v.Path.Logs, } return &model.Setting{ - Metadata: model.SettingsMetadata{ + Metadata: model.Metadata{ Category: "agent", Name: "task", Labels: util.MapStr{ From 05e1efa197c6dce66a247d0e77b70afa8c487a08 Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 19 Oct 2023 21:38:53 +0800 Subject: [PATCH 10/36] remove log --- modules/agent/api/elasticsearch.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/agent/api/elasticsearch.go b/modules/agent/api/elasticsearch.go index 2b54c739..1034abb6 100644 --- a/modules/agent/api/elasticsearch.go +++ b/modules/agent/api/elasticsearch.go @@ -192,8 +192,6 @@ func remoteConfigProvider(instance model.Instance) []*common.ConfigFile { } } - log.Error("remoteConfigProvider", "result", result) - return result } From bea9ca6d1c5c5d9aebf22cd30fc3f69371a957fc Mon Sep 17 00:00:00 2001 From: medcl Date: Fri, 20 Oct 2023 17:14:14 +0800 Subject: [PATCH 11/36] cleanup unused code --- modules/agent/api/elasticsearch.go | 527 +++++++++-------- modules/agent/api/init.go | 16 +- modules/agent/api/instance.go | 14 - modules/agent/api/remote_config.go | 165 ++++++ modules/agent/api/tod.go | 882 ----------------------------- modules/agent/common/helper.go | 455 --------------- modules/agent/model/config.go | 5 - modules/agent/model/task.go | 46 -- 8 files changed, 459 insertions(+), 1651 deletions(-) delete mode 100644 modules/agent/api/instance.go create mode 100644 modules/agent/api/remote_config.go delete mode 100644 modules/agent/api/tod.go delete mode 100644 modules/agent/common/helper.go delete mode 100644 modules/agent/model/task.go diff --git a/modules/agent/api/elasticsearch.go b/modules/agent/api/elasticsearch.go index 1034abb6..1926428a 100644 --- a/modules/agent/api/elasticsearch.go +++ b/modules/agent/api/elasticsearch.go @@ -7,16 +7,21 @@ package api import ( "bytes" "context" + "errors" "fmt" log "github.com/cihub/seelog" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" + "infini.sh/framework/core/global" "infini.sh/framework/core/model" "infini.sh/framework/core/orm" "infini.sh/framework/core/util" common2 "infini.sh/framework/modules/elastic/common" "infini.sh/framework/plugins/managed/common" + "infini.sh/framework/plugins/managed/server" "net/http" + "net/url" + "sync" "time" ) @@ -91,7 +96,6 @@ func refreshNodesInfo(inst *model.Instance) (*elastic.DiscoveryResult, error) { } for nodeID, node := range nodesInfo.Nodes { - v, ok := enrolledNodesByAgent[nodeID] if ok { node.ClusterID = v.ClusterID @@ -99,188 +103,14 @@ func refreshNodesInfo(inst *model.Instance) (*elastic.DiscoveryResult, error) { } } - ////not recognized by agent, need auth? - //for _, node := range nodesInfo.UnknownProcess{ - // for _, v := range node.ListenAddresses { - // //ask user to manual enroll this node - // //check local credentials, if it works, get node info - // } - //} - - // { - // //node was not recognized by agent, need auth? - // if node.HttpPort != "" { - // for _, v := range enrolledNodesByAgent { - // if v.PublishAddress != "" { - // if util.UnifyLocalAddress(v.PublishAddress) == util.UnifyLocalAddress(node.PublishAddress) { - // node.ClusterID = v.ClusterID - // node.ClusterName = v.ClusterName - // node.NodeUUID = v.NodeUUID - // node.ClusterUuid = v.ClusterUUID - // node.NodeName = v.NodeName - // node.Path.Home = v.PathHome - // node.Path.Logs = v.PathLogs - // node.AgentID = inst.ID - // //TODO verify node info if the node id really match, need to fetch the credentials for agent - // //or let manager sync configs to this agent, verify the node info after receiving the configs - // //report any error along with this agent and node info - // break - // } - // } - // } - // } - // - //} - return nodesInfo, nil } -type RemoteConfig struct { - orm.ORMObjectBase - Metadata model.Metadata `json:"metadata" elastic_mapping:"metadata: { type: object }"` - Payload common.ConfigFile `json:"payload" elastic_mapping:"payload: { type: object}"` -} - -func remoteConfigProvider(instance model.Instance) []*common.ConfigFile { - - //fetch configs from remote db - //fetch configs assigned to (instance=_all OR instance=$instance_id ) AND application.name=$application.name - - q := orm.Query{ - Size: 1000, - Conds: orm.And(orm.Eq("metadata.category", "app_settings"), - orm.Eq("metadata.name", instance.Application.Name), - orm.Eq("metadata.labels.instance", "_all"), - ), - } - - err, searchResult := orm.Search(RemoteConfig{}, &q) - if err != nil { - panic(err) - } - - result := []*common.ConfigFile{} - - for _, row := range searchResult.Result { - v, ok := row.(map[string]interface{}) - if ok { - x, ok := v["payload"] - if ok { - f, ok := x.(map[string]interface{}) - if ok { - name, ok := f["name"].(string) - if ok { - item := common.ConfigFile{} - item.Name = util.ToString(name) - item.Location = util.ToString(f["location"]) - item.Content = util.ToString(f["content"]) - item.Version,_ = util.ToInt64(util.ToString(f["version"])) - item.Size = int64(len(item.Content)) - item.Managed = true - t, ok := v["updated"] - if ok { - layout := "2006-01-02T15:04:05.999999-07:00" - t1, err := time.Parse(layout, util.ToString(t)) - if err == nil { - item.Updated = t1.Unix() - } - } - result=append(result,&item) - } - } - } - } - } - - return result -} - -func dynamicAgentConfigProvider(instance model.Instance) []*common.ConfigFile { - - //get config files from remote db - //get settings with this agent id - result := []*common.ConfigFile{} - ids, err := GetEnrolledNodesByAgent(&instance) - if err != nil { - panic(err) - } - - var latestTimestamp int64 - for _, v := range ids { - if v.Updated > latestTimestamp { - latestTimestamp = v.Updated - } - } - - if len(ids) > 0 { - - cfg := common.ConfigFile{} - cfg.Name = "generated_metrics_tasks.yml" - cfg.Location = "generated_metrics_tasks.yml" - cfg.Content = getConfigs(ids) - cfg.Size = int64(len(cfg.Content)) - cfg.Version = latestTimestamp - cfg.Managed = true - cfg.Updated = latestTimestamp - result = append(result, &cfg) - } - - return result -} - -func getConfigs(items map[string]BindingItem) string { - buffer := bytes.NewBuffer([]byte("configs.template:\n ")) - - for _, v := range items { - - if v.ClusterID == "" { - panic("cluster id is empty") - } - metadata := elastic.GetMetadata(v.ClusterID) - var clusterLevelEnabled = false - var nodeLevelEnabled = true - var clusterEndPoint = metadata.Config.GetAnyEndpoint() - credential, err := common2.GetCredential(metadata.Config.CredentialID) - if err != nil { - panic(err) - } - var dv interface{} - dv, err = credential.Decode() - if err != nil { - panic(err) - } - var username = "" - var password = "" - - if auth, ok := dv.(model.BasicAuth); ok { - username = auth.Username - password = auth.Password - } - - buffer.Write([]byte(fmt.Sprintf("- name: \"%v\"\n path: ./config/task_config.tpl\n "+ - "variable:\n "+ - "CLUSTER_ID: %v\n "+ - "CLUSTER_ENDPOINT: [\"%v\"]\n "+ - "CLUSTER_USERNAME: \"%v\"\n "+ - "CLUSTER_PASSWORD: \"%v\"\n "+ - "CLUSTER_LEVEL_TASKS_ENABLED: %v\n "+ - "NODE_LEVEL_TASKS_ENABLED: %v\n "+ - "NODE_LOGS_PATH: \"%v\"\n\n\n"+ - "#MANAGED_CONFIG_VERSION: %v\n"+ - "#MANAGED: true", - v.NodeUUID, v.ClusterID, clusterEndPoint, username, password, clusterLevelEnabled, nodeLevelEnabled, v.PathLogs, v.Updated))) - } - - //password: $[[keystore.$[[CLUSTER_ID]]_password]] - - return buffer.String() -} - //get nodes info via agent func GetElasticsearchNodesViaAgent(ctx context.Context, instance *model.Instance) (*elastic.DiscoveryResult, error) { req := &util.Request{ Method: http.MethodGet, - Path: "/elasticsearch/nodes/_discovery", + Path: "/elasticsearch/node/_discovery", Context: ctx, } @@ -293,37 +123,6 @@ func GetElasticsearchNodesViaAgent(ctx context.Context, instance *model.Instance return &obj, nil } -func AuthESNode(ctx context.Context, agentBaseURL string, cfg elastic.ElasticsearchConfig) (*model.ESNodeInfo, error) { - reqBody, err := util.ToJSONBytes(cfg) - if err != nil { - return nil, err - } - req := &util.Request{ - Method: http.MethodPost, - Path: "/elasticsearch/_auth", - Context: ctx, - Body: reqBody, - } - resBody := &model.ESNodeInfo{} - err = DoRequest(req, resBody) - if err != nil { - return nil, err - } - return resBody, nil -} - -func getNodeByPidOrUUID(nodes map[int]*model.ESNodeInfo, pid int, uuid string, port string) *model.ESNodeInfo { - if nodes[pid] != nil { - return nodes[pid] - } - for _, node := range nodes { - if node.NodeUUID != "" && node.NodeUUID == uuid { - return node - } - } - return nil -} - type BindingItem struct { ClusterName string `json:"cluster_name"` ClusterUUID string `json:"cluster_uuid"` @@ -338,50 +137,19 @@ type BindingItem struct { Updated int64 `json:"updated"` } -func getUnAssociateNodes() (map[string][]model.ESNodeInfo, error) { - query := util.MapStr{ - "size": 3000, - "query": util.MapStr{ - "bool": util.MapStr{ - "must_not": []util.MapStr{ - { - "exists": util.MapStr{ - "field": "cluster_id", - }, - }, - }, - }, - }, - } - q := orm.Query{ - RawQuery: util.MustToJSONBytes(query), - } - - err, result := orm.Search(model.ESNodeInfo{}, &q) - if err != nil { - return nil, err - } - nodesInfo := map[string][]model.ESNodeInfo{} - for _, row := range result.Result { - node := model.ESNodeInfo{} - buf := util.MustToJSONBytes(row) - util.MustFromJSONBytes(buf, &node) - nodesInfo[node.AgentID] = append(nodesInfo[node.AgentID], node) - } - return nodesInfo, nil -} - func GetElasticLogFiles(ctx context.Context, instance *model.Instance, logsPath string) (interface{}, error) { reqBody := util.MustToJSONBytes(util.MapStr{ "logs_path": logsPath, }) + req := &util.Request{ Method: http.MethodPost, Path: "/elasticsearch/logs/_list", Context: ctx, Body: reqBody, } + resBody := map[string]interface{}{} err := doRequest(instance, req, &resBody) if err != nil { @@ -391,6 +159,7 @@ func GetElasticLogFiles(ctx context.Context, instance *model.Instance, logsPath return nil, fmt.Errorf("get elasticsearch log files error: %v", resBody) } return resBody["result"], nil + } func GetElasticLogFileContent(ctx context.Context, instance *model.Instance, body interface{}) (interface{}, error) { @@ -535,3 +304,281 @@ func getAgentByNodeID(nodeID string) (*model.Instance, string, error) { } return nil, "", nil } + +type ClusterInfo struct { + ClusterIDs []string `json:"cluster_id"` +} + +func (h *APIHandler) discoveryESNodesInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + + id := ps.MustGetParameter("instance_id") + instance := model.Instance{} + instance.ID = id + exists, err := orm.Get(&instance) + if !exists || err != nil { + h.WriteJSON(w, util.MapStr{ + "_id": id, + "found": false, + }, http.StatusNotFound) + return + } + + nodes, err := refreshNodesInfo(&instance) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + if len(nodes.UnknownProcess) > 0 { + + var discoveredPIDs map[int]*elastic.LocalNodeInfo = make(map[int]*elastic.LocalNodeInfo) + + if req.Method == "POST" { + bytes, err := h.GetRawBody(req) + if err != nil { + panic(err) + } + if len(bytes) > 0 { + clusterInfo := ClusterInfo{} + util.FromJSONBytes(bytes, &clusterInfo) + if len(clusterInfo.ClusterIDs) > 0 { + //try connect this node to cluster by using this cluster's agent credential + for _, clusterID := range clusterInfo.ClusterIDs { + meta := elastic.GetMetadata(clusterID) + if meta != nil { + if meta.Config.AgentCredentialID != "" { + auth, err := common2.GetAgentBasicAuth(meta.Config) + if err != nil { + panic(err) + } + if auth != nil { + //try connect + for _, node := range nodes.UnknownProcess { + for _, v := range node.ListenAddresses { + ip := v.IP + if util.ContainStr(v.IP, "::") { + ip = fmt.Sprintf("[%s]", v.IP) + } + nodeHost := fmt.Sprintf("%s:%d", ip, v.Port) + success, tryAgain, nodeInfo := h.getESNodeInfoViaProxy(nodeHost, "http", auth, instance) + if !success && tryAgain { + //try https again + success, tryAgain, nodeInfo = h.getESNodeInfoViaProxy(nodeHost, "https", auth, instance) + } + if success { + log.Error("connect to es node success:", nodeHost) + discoveredPIDs[node.PID] = nodeInfo + break + } + } + } + } + } + } + } + } + } + } + + newUnknownProcess := []model.ProcessInfo{} + if len(discoveredPIDs) > 0 { + for _, node := range nodes.UnknownProcess { + if item, ok := discoveredPIDs[node.PID]; !ok { + newUnknownProcess = append(newUnknownProcess, node) + } else { + nodes.Nodes[item.NodeUUID] = item + } + } + nodes.UnknownProcess = newUnknownProcess + } + } + + h.WriteJSON(w, nodes, http.StatusOK) +} + +func (h *APIHandler) getESNodeInfoViaProxy(host string, schema string, auth *model.BasicAuth, instance model.Instance) (success, tryAgain bool, info *elastic.LocalNodeInfo) { + esConfig := elastic.ElasticsearchConfig{Host: host, Schema: schema, BasicAuth: auth} + body := util.MustToJSONBytes(esConfig) + res, err := server.ProxyRequestToRuntimeInstance(instance.GetEndpoint(), "POST", "/elasticsearch/node/_info", + body, int64(len(body)), auth) + + if err != nil { + panic(err) + } + + if global.Env().IsDebug { + if res != nil { // && res.StatusCode==http.StatusOK + log.Debug(string(res.Body)) + } + } + + if res != nil && res.StatusCode == http.StatusForbidden { + return false, false, nil + } + + if res != nil && res.StatusCode == http.StatusOK { + node := elastic.LocalNodeInfo{} + err := util.FromJSONBytes(res.Body, &node) + if err != nil { + panic(err) + } + return true, false, &node + } + + return false, true, nil +} + +func NewClusterSettings(clusterID string) *model.Setting { + settings := model.Setting{ + Metadata: model.Metadata{ + Category: Cluster, + }, + } + settings.ID = fmt.Sprintf("%v_%v_%v", settings.Metadata.Category, settings.Metadata.Name, clusterID) + + settings.Metadata.Labels = util.MapStr{ + "cluster_id": clusterID, + } + + return &settings +} + +func NewNodeAgentSettings(instanceID string, item *BindingItem) *model.Setting { + + settings := model.Setting{ + Metadata: model.Metadata{ + Category: Node, + Name: "agent", + }, + } + settings.ID = fmt.Sprintf("%v_%v_%v", settings.Metadata.Category, settings.Metadata.Name, item.NodeUUID) + + settings.Metadata.Labels = util.MapStr{ + "agent_id": instanceID, + } + + settings.Payload = util.MapStr{ + "cluster_id": item.ClusterID, + "cluster_name": item.ClusterName, + "cluster_uuid": item.ClusterUUID, + "node_uuid": item.NodeUUID, + "publish_address": item.PublishAddress, + "node_name": item.NodeName, + "path_home": item.PathHome, + "path_logs": item.PathLogs, + } + + return &settings +} + +func NewIndexSettings(clusterID, nodeID, agentID, indexName, indexID string) *model.Setting { + + settings := model.Setting{ + Metadata: model.Metadata{ + Category: Index, + }, + } + settings.ID = fmt.Sprintf("%v_%v_%v", settings.Metadata.Category, settings.Metadata.Name, nodeID) + + settings.Metadata.Labels = util.MapStr{ + "cluster_id": clusterID, + "node_id": nodeID, + "agent_id": agentID, + "index_name": indexName, + "index_id": indexID, + } + + return &settings +} + +const Cluster = "cluster_settings" +const Node = "node_settings" +const Index = "index_settings" + +func (h *APIHandler) revokeESNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + //agent id + instID := ps.MustGetParameter("instance_id") + item := BindingItem{} + err := h.DecodeJSON(req, &item) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + settings := NewNodeAgentSettings(instID, &item) + err = orm.Delete(&orm.Context{ + Refresh: "wait_for", + }, settings) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.WriteAckOKJSON(w) +} + +func (h *APIHandler) enrollESNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + + //agent id + instID := ps.MustGetParameter("instance_id") + + //node id and cluster id + item := BindingItem{} + err := h.DecodeJSON(req, &item) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + //update node's setting + settings := NewNodeAgentSettings(instID, &item) + err = orm.Update(&orm.Context{ + Refresh: "wait_for", + }, settings) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + h.WriteAckOKJSON(w) +} + +var mTLSClient *http.Client //TODO get mTLSClient +var initOnce = sync.Once{} + +func doRequest(instance *model.Instance, req *util.Request, obj interface{}) error { + var err error + var res *util.Result + + initOnce.Do(func() { + if global.Env().SystemConfig.Configs.TLSConfig.TLSEnabled && global.Env().SystemConfig.Configs.TLSConfig.TLSCAFile != "" { + + //init client + hClient, err := util.NewMTLSClient( + global.Env().SystemConfig.Configs.TLSConfig.TLSCAFile, + global.Env().SystemConfig.Configs.TLSConfig.TLSCertFile, + global.Env().SystemConfig.Configs.TLSConfig.TLSKeyFile) + if err != nil { + panic(err) + } + mTLSClient = hClient + } + }) + + req.Url, err = url.JoinPath(instance.GetEndpoint(), req.Path) + res, err = util.ExecuteRequestWithCatchFlag(mTLSClient, req, true) + if err != nil || res.StatusCode != 200 { + body := "" + if res != nil { + body = string(res.Body) + } + return errors.New(fmt.Sprintf("request error: %v, %v", err, body)) + } + + if res != nil { + if res.Body != nil { + return util.FromJSONBytes(res.Body, obj) + } + } + + return nil +} diff --git a/modules/agent/api/init.go b/modules/agent/api/init.go index d863e023..b1a25c7d 100644 --- a/modules/agent/api/init.go +++ b/modules/agent/api/init.go @@ -10,6 +10,10 @@ import ( "infini.sh/framework/plugins/managed/server" ) +type APIHandler struct { + api.Handler +} + func Init() { handler := APIHandler{} api.HandleAPIMethod(api.POST, "/host/_enroll", handler.enrollHost) @@ -18,23 +22,17 @@ func Init() { api.HandleAPIMethod(api.DELETE, "/host/:host_id", handler.deleteHost) //bind agent with nodes - api.HandleAPIMethod(api.GET, "/instance/:instance_id/node/_discovery", handler.RequirePermission(handler.getESNodesInfo, enum.PermissionAgentInstanceRead)) + api.HandleAPIMethod(api.GET, "/instance/:instance_id/node/_discovery", handler.RequirePermission(handler.discoveryESNodesInfo, enum.PermissionAgentInstanceRead)) + api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_discovery", handler.RequirePermission(handler.discoveryESNodesInfo, enum.PermissionAgentInstanceRead)) api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_enroll", handler.RequirePermission(handler.enrollESNode, enum.PermissionAgentInstanceWrite)) api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_revoke", handler.RequirePermission(handler.revokeESNode, enum.PermissionAgentInstanceWrite)) - api.HandleAPIMethod(api.POST, "/auto_associate", handler.RequirePermission(handler.autoAssociateESNode, enum.PermissionAgentInstanceWrite)) - - //api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_auth", handler.RequirePermission(handler.authESNode, enum.PermissionAgentInstanceWrite)) - //api.HandleAPIMethod(api.DELETE, "/instance/:instance_id/_nodes", handler.RequirePermission(handler.deleteESNode, enum.PermissionAgentInstanceWrite)) + //api.HandleAPIMethod(api.POST, "/auto_associate", handler.RequirePermission(handler.autoAssociateESNode, enum.PermissionAgentInstanceWrite)) //get elasticsearch node logs, direct fetch or via stored logs(TODO) api.HandleAPIMethod(api.GET, "/elasticsearch/:id/node/:node_id/logs/_list", handler.RequirePermission(handler.getLogFilesByNode, enum.PermissionAgentInstanceRead)) api.HandleAPIMethod(api.POST, "/elasticsearch/:id/node/:node_id/logs/_read", handler.RequirePermission(handler.getLogFileContent, enum.PermissionAgentInstanceRead)) - //api.HandleAPIMethod(api.POST, "/agent/install_command", handler.RequireLogin(handler.generateInstallCommand)) - //api.HandleAPIMethod(api.GET, "/agent/install.sh", handler.getInstallScript) - - server.RegisterConfigProvider(remoteConfigProvider) server.RegisterConfigProvider(dynamicAgentConfigProvider) } diff --git a/modules/agent/api/instance.go b/modules/agent/api/instance.go deleted file mode 100644 index 563ce8cf..00000000 --- a/modules/agent/api/instance.go +++ /dev/null @@ -1,14 +0,0 @@ -/* Copyright © INFINI Ltd. All rights reserved. - * Web: https://infinilabs.com - * Email: hello#infini.ltd */ - -package api - -import ( - "infini.sh/framework/core/api" -) - -type APIHandler struct { - api.Handler -} - diff --git a/modules/agent/api/remote_config.go b/modules/agent/api/remote_config.go new file mode 100644 index 00000000..e30fd7c2 --- /dev/null +++ b/modules/agent/api/remote_config.go @@ -0,0 +1,165 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package api + +import ( + "bytes" + "fmt" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + common2 "infini.sh/framework/modules/elastic/common" + "infini.sh/framework/plugins/managed/common" + "time" +) + +type RemoteConfig struct { + orm.ORMObjectBase + Metadata model.Metadata `json:"metadata" elastic_mapping:"metadata: { type: object }"` + Payload common.ConfigFile `json:"payload" elastic_mapping:"payload: { type: object}"` +} + +func remoteConfigProvider(instance model.Instance) []*common.ConfigFile { + + //fetch configs from remote db + //fetch configs assigned to (instance=_all OR instance=$instance_id ) AND application.name=$application.name + + q := orm.Query{ + Size: 1000, + Conds: orm.And(orm.Eq("metadata.category", "app_settings"), + orm.Eq("metadata.name", instance.Application.Name), + orm.Eq("metadata.labels.instance", "_all"), + ), + } + + err, searchResult := orm.Search(RemoteConfig{}, &q) + if err != nil { + panic(err) + } + + result := []*common.ConfigFile{} + + for _, row := range searchResult.Result { + v, ok := row.(map[string]interface{}) + if ok { + x, ok := v["payload"] + if ok { + f, ok := x.(map[string]interface{}) + if ok { + name, ok := f["name"].(string) + if ok { + item := common.ConfigFile{} + item.Name = util.ToString(name) + item.Location = util.ToString(f["location"]) + item.Content = util.ToString(f["content"]) + item.Version, _ = util.ToInt64(util.ToString(f["version"])) + item.Size = int64(len(item.Content)) + item.Managed = true + t, ok := v["updated"] + if ok { + layout := "2006-01-02T15:04:05.999999-07:00" + t1, err := time.Parse(layout, util.ToString(t)) + if err == nil { + item.Updated = t1.Unix() + } + } + result = append(result, &item) + } + } + } + } + } + + return result +} + +func dynamicAgentConfigProvider(instance model.Instance) []*common.ConfigFile { + + //get config files from remote db + //get settings with this agent id + + result := []*common.ConfigFile{} + ids, err := GetEnrolledNodesByAgent(&instance) + if err != nil { + panic(err) + } + + var latestTimestamp int64 + for _, v := range ids { + if v.Updated > latestTimestamp { + latestTimestamp = v.Updated + } + } + + if len(ids) > 0 { + + cfg := common.ConfigFile{} + cfg.Name = "generated_metrics_tasks.yml" + cfg.Location = "generated_metrics_tasks.yml" + cfg.Content = getAgentIngestConfigs(ids) + cfg.Size = int64(len(cfg.Content)) + cfg.Version = latestTimestamp + cfg.Managed = true + cfg.Updated = latestTimestamp + result = append(result, &cfg) + } + + return result +} + +func getAgentIngestConfigs(items map[string]BindingItem) string { + + buffer := bytes.NewBuffer([]byte("configs.template:\n ")) + + for _, v := range items { + + if v.ClusterID == "" { + panic("cluster id is empty") + } + + metadata := elastic.GetMetadata(v.ClusterID) + var clusterLevelEnabled = false + var nodeLevelEnabled = true + var clusterEndPoint = metadata.Config.GetAnyEndpoint() + + var username = "" + var password = "" + + if metadata.Config.AgentCredentialID != "" { + credential, err := common2.GetCredential(metadata.Config.AgentCredentialID) + if err != nil { + panic(err) + } + var dv interface{} + dv, err = credential.Decode() + if err != nil { + panic(err) + } + if auth, ok := dv.(model.BasicAuth); ok { + username = auth.Username + password = auth.Password + } + } + + buffer.Write([]byte(fmt.Sprintf("- name: \"%v\"\n path: ./config/task_config.tpl\n "+ + "variable:\n "+ + "CLUSTER_ID: %v\n "+ + "CLUSTER_ENDPOINT: [\"%v\"]\n "+ + "CLUSTER_USERNAME: \"%v\"\n "+ + "CLUSTER_PASSWORD: \"%v\"\n "+ + "CLUSTER_LEVEL_TASKS_ENABLED: %v\n "+ + "NODE_LEVEL_TASKS_ENABLED: %v\n "+ + "NODE_LOGS_PATH: \"%v\"\n\n\n"+ + "#MANAGED_CONFIG_VERSION: %v\n"+ + "#MANAGED: true", + v.NodeUUID, v.ClusterID, clusterEndPoint, username, password, clusterLevelEnabled, nodeLevelEnabled, v.PathLogs, v.Updated))) + } + + //password: $[[keystore.$[[CLUSTER_ID]]_password]] + + return buffer.String() +} + diff --git a/modules/agent/api/tod.go b/modules/agent/api/tod.go deleted file mode 100644 index da6c172f..00000000 --- a/modules/agent/api/tod.go +++ /dev/null @@ -1,882 +0,0 @@ -/* Copyright © INFINI LTD. All rights reserved. - * Web: https://infinilabs.com - * Email: hello#infini.ltd */ - -package api - -import ( - "context" - "errors" - "fmt" - log "github.com/cihub/seelog" - common2 "infini.sh/console/modules/agent/common" - model3 "infini.sh/console/modules/agent/model" - httprouter "infini.sh/framework/core/api/router" - "infini.sh/framework/core/elastic" - "infini.sh/framework/core/global" - "infini.sh/framework/core/host" - "infini.sh/framework/core/model" - "infini.sh/framework/core/orm" - "infini.sh/framework/core/util" - "infini.sh/framework/modules/elastic/common" - "net/http" - "net/url" - "strings" - "sync" -) - -//func (h *APIHandler) updateInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { -// id := ps.MustGetParameter("instance_id") -// oldInst := model.Instance{} -// oldInst.ID = id -// _, err := orm.Get(&oldInst) -// if err != nil { -// if err == elastic2.ErrNotFound { -// h.WriteJSON(w, util.MapStr{ -// "_id": id, -// "result": "not_found", -// }, http.StatusNotFound) -// return -// } -// h.WriteError(w, err.Error(), http.StatusInternalServerError) -// log.Error(err) -// return -// } -// -// obj := model.Instance{} -// err = h.DecodeJSON(req, &obj) -// if err != nil { -// h.WriteError(w, err.Error(), http.StatusInternalServerError) -// log.Error(err) -// return -// } -// -// oldInst.Name = obj.Name -// oldInst.Endpoint = obj.Endpoint -// oldInst.Description = obj.Description -// oldInst.Tags = obj.Tags -// oldInst.BasicAuth = obj.BasicAuth -// err = orm.Update(&orm.Context{ -// Refresh: "wait_for", -// }, &oldInst) -// if err != nil { -// h.WriteError(w, err.Error(), http.StatusInternalServerError) -// log.Error(err) -// return -// } -// -// h.WriteJSON(w, util.MapStr{ -// "_id": obj.ID, -// "result": "updated", -// }, 200) -//} -// -//func (h *APIHandler) searchInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { -// -// var ( -// keyword = h.GetParameterOrDefault(req, "keyword", "") -// //queryDSL = `{"query":{"bool":{"must":[%s]}}, "size": %d, "from": %d}` -// strSize = h.GetParameterOrDefault(req, "size", "20") -// strFrom = h.GetParameterOrDefault(req, "from", "0") -// ) -// -// var ( -// mustQ []interface{} -// ) -// -// if keyword != "" { -// mustQ = append(mustQ, util.MapStr{ -// "query_string": util.MapStr{ -// "default_field": "*", -// "query": keyword, -// }, -// }) -// } -// size, _ := strconv.Atoi(strSize) -// if size <= 0 { -// size = 20 -// } -// from, _ := strconv.Atoi(strFrom) -// if from < 0 { -// from = 0 -// } -// -// queryDSL := util.MapStr{ -// "size": size, -// "from": from, -// } -// if len(mustQ) > 0 { -// queryDSL["query"] = util.MapStr{ -// "bool": util.MapStr{ -// "must": mustQ, -// }, -// } -// } -// -// q := orm.Query{} -// q.RawQuery = util.MustToJSONBytes(queryDSL) -// -// err, res := orm.Search(&model.Instance{}, &q) -// if err != nil { -// log.Error(err) -// h.WriteError(w, err.Error(), http.StatusInternalServerError) -// return -// } -// -// h.Write(w, res.Raw) -//} - -func (h *APIHandler) getESNodesInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - - id := ps.MustGetParameter("instance_id") - obj := model.Instance{} - obj.ID = id - exists, err := orm.Get(&obj) - if !exists || err != nil { - h.WriteJSON(w, util.MapStr{ - "_id": id, - "found": false, - }, http.StatusNotFound) - return - } - - nodes, err := refreshNodesInfo(&obj) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - h.WriteJSON(w, nodes, http.StatusOK) -} - -func (h *APIHandler) authESNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - id := ps.MustGetParameter("instance_id") - inst := model.Instance{} - inst.ID = id - exists, err := orm.Get(&inst) - if !exists || err != nil { - h.WriteJSON(w, util.MapStr{ - "_id": id, - "found": false, - }, http.StatusNotFound) - return - } - reqBody := struct { - NodeID string `json:"node_id"` - ESConfig *elastic.ElasticsearchConfig `json:"es_config"` - }{} - err = h.DecodeJSON(req, &reqBody) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - //node id maybe is missing - if reqBody.NodeID != "" { - //verify the node id, if the node is is actually the node of the instance - oldNodeInfo := &model.ESNodeInfo{ - ID: reqBody.NodeID, - } - exists, err = orm.Get(oldNodeInfo) - if !exists || err != nil { - h.WriteJSON(w, fmt.Sprintf("node [%s] of agent [%s] was not found", oldNodeInfo.ID, inst.Name), http.StatusInternalServerError) - return - } - } else { - //find out the node id with credentials - cfg := reqBody.ESConfig - if cfg.Endpoint == "" { - cfg.Endpoint = cfg.GetAnyEndpoint() - } - basicAuth, err := common.GetBasicAuth(cfg) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - cfg.BasicAuth = basicAuth - nodeInfo, err := AuthESNode(context.Background(), inst.GetEndpoint(), *cfg) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - //host, port, err := net.SplitHostPort(nodeInfo.PublishAddress) - //if err != nil { - // log.Error(err) - // h.WriteError(w, err.Error(), http.StatusInternalServerError) - // return - //} - //if !util.StringInArray(inst.Network.IP, host) && !net.ParseIP(host).IsLoopback() { - // h.WriteError(w, fmt.Sprintf("got node host %s not match any ip of %v", host, inst.Network.IP), http.StatusInternalServerError) - // return - //} - //if oldNodeInfo.HttpPort != port { - // h.WriteError(w, fmt.Sprintf("port mismatch, got: %s，expected: %s", port, oldNodeInfo.HttpPort), http.StatusInternalServerError) - // return - //} - //if oldNodeInfo.ProcessInfo.PID != nodeInfo.ProcessInfo.PID { - // h.WriteError(w, fmt.Sprintf("process id mismatch, got: %d，expected: %d", nodeInfo.ProcessInfo.PID, oldNodeInfo.ProcessInfo.PID), http.StatusInternalServerError) - // return - //} - - reqBody.NodeID = nodeInfo.NodeUUID - } - - //nodeInfo:=elastic.NodeConfig{} - //nodeInfo.ID = reqBody.NodeID - //nodeInfo.AgentID = inst.ID - //err = orm.Update(nil, nodeInfo) //update node's info and agent_id - //if err != nil { - // log.Error(err) - // h.WriteError(w, err.Error(), http.StatusInternalServerError) - // return - //} - //h.WriteJSON(w, nodeInfo, http.StatusOK) -} - -func NewClusterSettings(clusterID string) *model.Setting { - settings := model.Setting{ - Metadata: model.Metadata{ - Category: Cluster, - }, - } - settings.ID = fmt.Sprintf("%v_%v_%v", settings.Metadata.Category, settings.Metadata.Name, clusterID) - - settings.Metadata.Labels = util.MapStr{ - "cluster_id": clusterID, - } - - return &settings -} - -func NewNodeAgentSettings(instanceID string, item *BindingItem) *model.Setting { - - settings := model.Setting{ - Metadata: model.Metadata{ - Category: Node, - Name: "agent", - }, - } - settings.ID = fmt.Sprintf("%v_%v_%v", settings.Metadata.Category, settings.Metadata.Name, item.NodeUUID) - - settings.Metadata.Labels = util.MapStr{ - "agent_id": instanceID, - } - - settings.Payload = util.MapStr{ - "cluster_id": item.ClusterID, - "cluster_name": item.ClusterName, - "cluster_uuid": item.ClusterUUID, - "node_uuid": item.NodeUUID, - "publish_address": item.PublishAddress, - "node_name": item.NodeName, - "path_home": item.PathHome, - "path_logs": item.PathLogs, - } - - return &settings -} - -func NewIndexSettings(clusterID, nodeID, agentID, indexName, indexID string) *model.Setting { - - settings := model.Setting{ - Metadata: model.Metadata{ - Category: Index, - }, - } - settings.ID = fmt.Sprintf("%v_%v_%v", settings.Metadata.Category, settings.Metadata.Name, nodeID) - - settings.Metadata.Labels = util.MapStr{ - "cluster_id": clusterID, - "node_id": nodeID, - "agent_id": agentID, - "index_name": indexName, - "index_id": indexID, - } - - return &settings -} - -const Cluster = "cluster_settings" -const Node = "node_settings" -const Index = "index_settings" - -func (h *APIHandler) revokeESNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - //agent id - instID := ps.MustGetParameter("instance_id") - item := BindingItem{} - err := h.DecodeJSON(req, &item) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - settings := NewNodeAgentSettings(instID, &item) - err = orm.Delete(&orm.Context{ - Refresh: "wait_for", - }, settings) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - h.WriteAckOKJSON(w) -} - -func (h *APIHandler) enrollESNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - - //agent id - instID := ps.MustGetParameter("instance_id") - - //node id and cluster id - item := BindingItem{} - err := h.DecodeJSON(req, &item) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - //update node's setting - settings := NewNodeAgentSettings(instID, &item) - err = orm.Update(&orm.Context{ - Refresh: "wait_for", - }, settings) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - h.WriteAckOKJSON(w) -} - -func (h *APIHandler) autoAssociateESNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - reqBody := struct { - ClusterIDs []string `json:"cluster_ids"` - }{} - err := h.DecodeJSON(req, &reqBody) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - // query not associated nodes info - nodesM, err := getUnAssociateNodes() - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - if len(nodesM) == 0 { - h.WriteAckOKJSON(w) - return - } - agentIds := make([]string, 0, len(nodesM)) - for agentID := range nodesM { - agentIds = append(agentIds, agentID) - } - agents, err := getAgentByIds(agentIds) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - for _, clusterID := range reqBody.ClusterIDs { - // query cluster basicauth - cfg := elastic.GetConfig(clusterID) - basicAuth, err := common.GetBasicAuth(cfg) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - taskSetting, err := getSettingsByClusterID(cfg.ID) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - for agentID, nodes := range nodesM { - var ( - inst *model.Instance - ok bool - ) - if inst, ok = agents[agentID]; !ok { - log.Warnf("agent [%v] was not found", agentID) - continue - } - settings, err := common2.GetAgentSettings(agentID, 0) - if err != nil { - log.Error(err) - continue - } - for _, v := range nodes { - host := v.PublishAddress - var endpoint string - if strings.HasPrefix(host, "::") { //for ipv6 - instURL, err := url.Parse(inst.Endpoint) - if err != nil { - log.Error(err) - continue - } - host = instURL.Hostname() - endpoint = fmt.Sprintf("%s://[%s]:%s", v.Schema, host, v.HttpPort) - } else { - endpoint = fmt.Sprintf("%s://%s", v.Schema, host) - } - escfg := elastic.ElasticsearchConfig{ - Endpoint: endpoint, - BasicAuth: basicAuth, - } - nodeInfo, err := AuthESNode(context.Background(), inst.GetEndpoint(), escfg) - if err != nil { - log.Warn(err) - continue - } - //matched - if nodeInfo.ClusterUuid == cfg.ClusterUUID { - //update node info - nodeInfo.ID = v.ID - nodeInfo.AgentID = inst.ID - nodeInfo.ClusterID = cfg.ID - err = orm.Save(nil, nodeInfo) - if err != nil { - log.Error(err) - continue - } - setting := pickAgentSettings(settings, v) - if setting == nil { - tsetting := model3.TaskSetting{ - NodeStats: &model3.NodeStatsTask{ - Enabled: true, - }, - Logs: &model3.LogsTask{ - Enabled: true, - LogsPath: nodeInfo.Path.Logs, - }, - } - if taskSetting.IndexStats != nil { - tsetting.IndexStats = taskSetting.IndexStats - taskSetting.IndexStats = nil - } - if taskSetting.ClusterHealth != nil { - tsetting.ClusterHealth = taskSetting.ClusterHealth - taskSetting.ClusterHealth = nil - } - if taskSetting.ClusterStats != nil { - tsetting.ClusterStats = taskSetting.ClusterStats - taskSetting.ClusterStats = nil - } - setting = &model.Setting{ - Metadata: model.Metadata{ - Category: "agent", - Name: "task", - Labels: util.MapStr{ - "agent_id": agentID, - "cluster_uuid": nodeInfo.ClusterUuid, - "cluster_id": nodeInfo.ClusterID, - "node_uuid": nodeInfo.NodeUUID, - "endpoint": fmt.Sprintf("%s://%s", nodeInfo.Schema, nodeInfo.PublishAddress), - }, - }, - Payload: util.MapStr{ - "task": tsetting, - }, - } - err = orm.Create(nil, setting) - if err != nil { - log.Error("save agent task setting error: ", err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - } - } - } - - } - } - h.WriteAckOKJSON(w) -} - -func getAgentByIds(agentIDs []string) (map[string]*model.Instance, error) { - query := util.MapStr{ - "size": len(agentIDs), - "query": util.MapStr{ - "terms": util.MapStr{ - "id": agentIDs, - }, - }, - } - q := orm.Query{ - RawQuery: util.MustToJSONBytes(query), - } - err, result := orm.Search(model.Instance{}, &q) - if err != nil { - return nil, err - } - agents := map[string]*model.Instance{} - for _, row := range result.Result { - inst := model.Instance{} - buf := util.MustToJSONBytes(row) - util.MustFromJSONBytes(buf, &inst) - agents[inst.ID] = &inst - } - return agents, nil -} - -func (h *APIHandler) deleteESNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - id := ps.MustGetParameter("instance_id") - nodeIDs := []string{} - err := h.DecodeJSON(req, &nodeIDs) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - if len(nodeIDs) > 0 { - q := util.MapStr{ - "query": util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "terms": util.MapStr{ - "id": nodeIDs, - }, - }, - { - "term": util.MapStr{ - "agent_id": util.MapStr{ - "value": id, - }, - }, - }, - }, - }, - }, - } - err = orm.DeleteBy(model.ESNodeInfo{}, util.MustToJSONBytes(q)) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - q = util.MapStr{ - "query": util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "terms": util.MapStr{ - "metadata.labels.node_uuid": nodeIDs, - }, - }, - { - "term": util.MapStr{ - "metadata.labels.agent_id": util.MapStr{ - "value": id, - }, - }, - }, - }, - }, - }, - } - } - h.WriteAckOKJSON(w) -} - -// -//func (h *APIHandler) tryConnect(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { -// var reqBody = struct { -// Endpoint string `json:"endpoint"` -// BasicAuth model.BasicAuth -// }{} -// err := h.DecodeJSON(req, &reqBody) -// if err != nil { -// h.WriteError(w, err.Error(), http.StatusInternalServerError) -// return -// } -// connectRes, err := client.GetClient().GetInstanceBasicInfo(context.Background(), reqBody.Endpoint) -// if err != nil { -// h.WriteError(w, err.Error(), http.StatusInternalServerError) -// return -// } -// h.WriteJSON(w, connectRes, http.StatusOK) -//} - -func pickAgentSettings(settings []model.Setting, nodeInfo model.ESNodeInfo) *model.Setting { - for _, setting := range settings { - if setting.Metadata.Labels["node_uuid"] == nodeInfo.NodeUUID { - return &setting - } - } - return nil -} - -func getAgentTaskSetting(agentID string, v model.ESNodeInfo) (*model.Setting, error) { - taskSetting, err := getSettingsByClusterID(v.ClusterID) - if err != nil { - return nil, err - } - taskSetting.Logs = &model3.LogsTask{ - Enabled: true, - LogsPath: v.Path.Logs, - } - return &model.Setting{ - Metadata: model.Metadata{ - Category: "agent", - Name: "task", - Labels: util.MapStr{ - "agent_id": agentID, - "cluster_uuid": v.ClusterUuid, - "cluster_id": v.ClusterID, - "node_uuid": v.NodeUUID, - "endpoint": fmt.Sprintf("%s://%s", v.Schema, v.PublishAddress), - }, - }, - Payload: util.MapStr{ - "task": taskSetting, - }, - }, nil -} - -// getSettingsByClusterID query agent task settings with cluster id -func getSettingsByClusterID(clusterID string) (*model3.TaskSetting, error) { - err, result := querySettingsByClusterID(clusterID) - if err != nil { - return nil, err - } - - setting := &model3.TaskSetting{ - NodeStats: &model3.NodeStatsTask{ - Enabled: true, - }, - } - var ( - clusterStats = true - indexStats = true - clusterHealth = true - ) - keys := []string{"payload.task.cluster_stats.enabled", "payload.task.cluster_health.enabled", "payload.task.index_stats.enabled"} - for _, row := range result.Result { - if v, ok := row.(map[string]interface{}); ok { - vm := util.MapStr(v) - for _, key := range keys { - tv, _ := vm.GetValue(key) - if tv == true { - switch key { - case "payload.task.cluster_stats.enabled": - clusterStats = false - case "payload.task.index_stats.enabled": - indexStats = false - case "payload.task.cluster_health.enabled": - clusterHealth = false - } - } - } - } - } - if clusterStats { - setting.ClusterStats = &model3.ClusterStatsTask{ - Enabled: true, - } - } - if indexStats { - setting.IndexStats = &model3.IndexStatsTask{ - Enabled: true, - } - } - if clusterHealth { - setting.ClusterHealth = &model3.ClusterHealthTask{ - Enabled: true, - } - } - return setting, nil -} - -func querySettingsByClusterID(clusterID string) (error, orm.Result) { - queryDsl := util.MapStr{ - "size": 500, - "query": util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "term": util.MapStr{ - "metadata.labels.cluster_id": util.MapStr{ - "value": clusterID, - }, - }, - }, - }, - "minimum_should_match": 1, - "should": []util.MapStr{ - { - "term": util.MapStr{ - "payload.task.cluster_health.enabled": util.MapStr{ - "value": true, - }, - }, - }, - { - "term": util.MapStr{ - "payload.task.cluster_stats.enabled": util.MapStr{ - "value": true, - }, - }, - }, - { - "term": util.MapStr{ - "payload.task.index_stats.enabled": util.MapStr{ - "value": true, - }, - }, - }, - }, - }, - }, - } - q := orm.Query{ - RawQuery: util.MustToJSONBytes(queryDsl), - } - return orm.Search(model.Setting{}, &q) -} - -func GetHostInfo(ctx context.Context, agentBaseURL string) (*host.HostInfo, error) { - req := &util.Request{ - Method: http.MethodGet, - Path: "/agent/host/_basic", - Context: ctx, - } - resBody := struct { - Success bool `json:"success"` - Error string `json:"error"` - HostInfo *host.HostInfo `json:"result"` - }{} - - req.Body = util.MustToJSONBytes(resBody) - - err := DoRequest(req, &resBody) - if err != nil { - return nil, err - } - - if resBody.Success != true { - return nil, fmt.Errorf("enroll error from client: %v", resBody.Error) - } - return resBody.HostInfo, nil -} - -func GetElasticProcess(ctx context.Context, agentBaseURL string, agentID string) (interface{}, error) { - req := &util.Request{ - Method: http.MethodGet, - Path: fmt.Sprintf("/elasticsearch/%s/process/_elastic", agentID), - Context: ctx, - } - resBody := map[string]interface{}{} - err := DoRequest(req, &resBody) - if err != nil { - return nil, err - } - if resBody["success"] != true { - return nil, fmt.Errorf("discover host callback error: %v", resBody["error"]) - } - return resBody["elastic_process"], nil -} - -func GetInstanceBasicInfo(ctx context.Context, agentBaseURL string) (*model.Instance, error) { - req := &util.Request{ - Method: http.MethodGet, - Path: "/_info", - Context: ctx, - } - resBody := &model.Instance{} - err := DoRequest(req, &resBody) - return resBody, err -} - -func RegisterElasticsearch(ctx context.Context, agentBaseURL string, cfgs []elastic.ElasticsearchConfig) error { - reqBody, err := util.ToJSONBytes(cfgs) - if err != nil { - return err - } - req := &util.Request{ - Method: http.MethodPost, - Path: "/elasticsearch/_register", - Context: ctx, - Body: reqBody, - } - resBody := util.MapStr{} - err = DoRequest(req, &resBody) - if err != nil { - return err - } - if resBody["acknowledged"] != true { - return fmt.Errorf("%v", resBody["error"]) - } - return nil -} - -func CreatePipeline(ctx context.Context, agentBaseURL string, body []byte) error { - req := &util.Request{ - Method: http.MethodPost, - Path: "/pipeline/tasks/", - Body: body, - Context: ctx, - } - resBody := util.MapStr{} - return DoRequest(req, &resBody) -} - -func DeletePipeline(ctx context.Context, agentBaseURL, pipelineID string) error { - req := &util.Request{ - Method: http.MethodDelete, - Path: fmt.Sprintf("/pipeline/task/%s", pipelineID), - Context: ctx, - } - return DoRequest(req, nil) -} - -func DoRequest(req *util.Request, obj interface{}) error { - panic("implement me") -} - -var mTLSClient *http.Client //TODO get mTLSClient -var initOnce = sync.Once{} - -func doRequest(instance *model.Instance, req *util.Request, obj interface{}) error { - var err error - var res *util.Result - - initOnce.Do(func() { - if global.Env().SystemConfig.Configs.TLSConfig.TLSEnabled && global.Env().SystemConfig.Configs.TLSConfig.TLSCAFile != "" { - - //init client - hClient, err := util.NewMTLSClient( - global.Env().SystemConfig.Configs.TLSConfig.TLSCAFile, - global.Env().SystemConfig.Configs.TLSConfig.TLSCertFile, - global.Env().SystemConfig.Configs.TLSConfig.TLSKeyFile) - if err != nil { - panic(err) - } - mTLSClient = hClient - } - }) - - req.Url, err = url.JoinPath(instance.GetEndpoint(), req.Path) - res, err = util.ExecuteRequestWithCatchFlag(mTLSClient, req, true) - if err != nil || res.StatusCode != 200 { - body := "" - if res != nil { - body = string(res.Body) - } - return errors.New(fmt.Sprintf("request error: %v, %v", err, body)) - } - - if res != nil { - if res.Body != nil { - return util.FromJSONBytes(res.Body, obj) - } - } - - return nil -} diff --git a/modules/agent/common/helper.go b/modules/agent/common/helper.go deleted file mode 100644 index 7baa7952..00000000 --- a/modules/agent/common/helper.go +++ /dev/null @@ -1,455 +0,0 @@ -/* Copyright © INFINI Ltd. All rights reserved. - * Web: https://infinilabs.com - * Email: hello#infini.ltd */ - -package common - -import ( - "fmt" - log "github.com/cihub/seelog" - model2 "infini.sh/console/modules/agent/model" - "infini.sh/framework/core/credential" - "infini.sh/framework/core/elastic" - "infini.sh/framework/core/event" - "infini.sh/framework/core/global" - "infini.sh/framework/core/model" - "infini.sh/framework/core/orm" - "infini.sh/framework/core/util" - "strings" -) - -func ParseAgentSettings(settings []model.Setting)(*model2.ParseAgentSettingsResult, error){ - var clusterCfgs []elastic.ElasticsearchConfig - var ( - pipelines []util.MapStr - toDeletePipelineNames []string - ) - for _, setting := range settings { - if setting.Metadata.Labels == nil { - return nil, fmt.Errorf("empty metadata labels of setting [%s]", setting.ID) - } - var ( - clusterID string - ok bool - ) - nodeUUID := util.ToString(setting.Metadata.Labels["node_uuid"]) - if clusterID, ok = setting.Metadata.Labels["cluster_id"].(string); ok && clusterID != ""{ - cfg := elastic.GetConfig(clusterID) - newID := getClusterConfigReferenceName(clusterID, nodeUUID) - newCfg := elastic.ElasticsearchConfig{ - Enabled: true, - Name: newID, - BasicAuth: cfg.BasicAuth, - //todo get endpoint from agent node info - Endpoint: setting.Metadata.Labels["endpoint"].(string), - ClusterUUID: cfg.ClusterUUID, - } - newCfg.ID = newID - clusterCfgs = append(clusterCfgs, newCfg) - }else{ - return nil, fmt.Errorf("got wrong cluster id [%v] from metadata labels", setting.Metadata.Labels["cluster_id"]) - } - - taskCfg, err := util.MapStr(setting.Payload).GetValue("task") - if err != nil { - return nil, err - } - vBytes, err := util.ToJSONBytes(taskCfg) - if err != nil { - return nil, err - } - taskSetting := model2.TaskSetting{} - err = util.FromJSONBytes(vBytes, &taskSetting) - if err != nil { - return nil, err - } - partPipelines, partDeletePipelineNames, err := TransformSettingsToConfig(&taskSetting, clusterID, nodeUUID) - if err != nil { - return nil, err - } - pipelines = append(pipelines, partPipelines...) - toDeletePipelineNames = append(toDeletePipelineNames, partDeletePipelineNames...) - } - return &model2.ParseAgentSettingsResult{ - ClusterConfigs: clusterCfgs, - Pipelines: pipelines, - ToDeletePipelineNames: toDeletePipelineNames, - }, nil -} - -// GetAgentSettings query agent setting by agent id and updated timestamp, -// if there has any setting was updated, then return setting list includes settings not changed, -// otherwise return empty setting list -func GetAgentSettings(agentID string, timestamp int64) ([]model.Setting, error) { - query := util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "term": util.MapStr{ - "metadata.category": util.MapStr{ - "value": "agent", - }, - }, - }, - { - "term": util.MapStr{ - "metadata.name": util.MapStr{ - "value": "task", - }, - }, - }, - { - "term": util.MapStr{ - "metadata.labels.agent_id": util.MapStr{ - "value": agentID, - }, - }, - }, - //{ - // "range": util.MapStr{ - // "updated": util.MapStr{ - // "gt": timestamp, - // }, - // }, - //}, - }, - }, - } - queryDsl := util.MapStr{ - "size": 1000, - "query": query, - } - q := orm.Query{ - RawQuery: util.MustToJSONBytes(queryDsl), - } - err, result := orm.Search(model.Setting{}, &q) - if err != nil { - return nil, fmt.Errorf("search settings error: %w", err) - } - if len(result.Result) == 0 { - return nil, nil - } - var ( - settings []model.Setting - hasUpdated bool - ) - for _, row := range result.Result { - setting := model.Setting{} - buf, err := util.ToJSONBytes(row) - if err != nil { - return nil, err - } - err = util.FromJSONBytes(buf, &setting) - if err != nil { - return nil, err - } - if setting.Updated != nil && setting.Updated.UnixMilli() > timestamp { - hasUpdated = true - } - settings = append(settings, setting) - } - if !hasUpdated { - return nil, nil - } - return settings, nil -} - -func getClusterConfigReferenceName(clusterID, nodeUUID string) string { - return fmt.Sprintf("%s_%s", clusterID, nodeUUID) -} - -func TransformSettingsToConfig(setting *model2.TaskSetting, clusterID, nodeUUID string) ([]util.MapStr, []string, error) { - if setting == nil { - return nil, nil, fmt.Errorf("empty setting") - } - var ( - pipelines []util.MapStr - toDeletePipelineNames []string - ) - if setting.ClusterStats != nil { - var processorName = "es_cluster_stats" - if setting.ClusterStats.Enabled { - pipelineCfg, err := newClusterMetricPipeline(processorName, clusterID, nodeUUID) - if err != nil { - return nil, nil, err - } - pipelines = append(pipelines, pipelineCfg) - }else{ - toDeletePipelineNames = append(toDeletePipelineNames, getMetricPipelineName(clusterID, processorName)) - } - } - if setting.IndexStats != nil { - var processorName = "es_index_stats" - if setting.IndexStats.Enabled { - pipelineCfg, err := newClusterMetricPipeline(processorName, clusterID, nodeUUID) - if err != nil { - return nil, nil, err - } - pipelines = append(pipelines, pipelineCfg) - }else{ - toDeletePipelineNames = append(toDeletePipelineNames, getMetricPipelineName(clusterID, processorName)) - } - } - if setting.ClusterHealth != nil { - var processorName = "es_cluster_health" - if setting.ClusterHealth.Enabled { - pipelineCfg, err := newClusterMetricPipeline(processorName, clusterID, nodeUUID) - if err != nil { - return nil, nil, err - } - pipelines = append(pipelines, pipelineCfg) - }else{ - toDeletePipelineNames = append(toDeletePipelineNames, getMetricPipelineName(clusterID, processorName)) - } - } - if setting.NodeStats != nil { - var processorName = "es_node_stats" - if setting.NodeStats.Enabled { - params := util.MapStr{ - "elasticsearch": getClusterConfigReferenceName(clusterID, nodeUUID), - "labels": util.MapStr{ - "cluster_id": clusterID, - }, - } - if len(setting.NodeStats.NodeIDs) > 0{ - params["node_uuids"] = setting.NodeStats.NodeIDs - } - cfg := util.MapStr{ - processorName: params, - } - enabled := true - pipelineCfg := util.MapStr{ - "enabled": &enabled, - "name": getMetricPipelineName(nodeUUID, processorName), - "auto_start": true, - "keep_running": true, - "retry_delay_in_ms": 10000, - "processor": []util.MapStr{cfg}, - } - pipelines = append(pipelines, pipelineCfg) - }else{ - toDeletePipelineNames = append(toDeletePipelineNames, getMetricPipelineName(nodeUUID, processorName)) - } - } - if setting.Logs != nil { - var processorName = "es_logs_processor" - if setting.Logs.Enabled { - params := util.MapStr{ - "elasticsearch": getClusterConfigReferenceName(clusterID, nodeUUID), - "queue_name": "logs", - "labels": util.MapStr{ - "cluster_id": clusterID, - }, - } - if setting.Logs.LogsPath != "" { - params["logs_path"] = setting.Logs.LogsPath - } - cfg := util.MapStr{ - processorName: params, - } - enabled := true - pipelineCfg := util.MapStr{ - "enabled": &enabled, - "name": fmt.Sprintf("collect_%s_es_logs", nodeUUID), - "auto_start": true, - "keep_running": true, - "retry_delay_in_ms": 3000, - "processor": []util.MapStr{cfg}, - } - pipelines = append(pipelines, pipelineCfg) - } - } - return pipelines, toDeletePipelineNames, nil -} - - -func newClusterMetricPipeline(processorName string, clusterID string, nodeUUID string)(util.MapStr, error){ - referName := getClusterConfigReferenceName(clusterID, nodeUUID) - cfg := util.MapStr{ - processorName: util.MapStr{ - "elasticsearch": referName, - "labels": util.MapStr{ - "cluster_id": clusterID, - }, - }, - } - enabled := true - pipelineCfg := util.MapStr{ - "enabled": &enabled, - "name": getMetricPipelineName(clusterID, processorName), - "auto_start": true, - "keep_running": true, - "singleton": true, - "retry_delay_in_ms": 10000, - "processor": []util.MapStr{cfg}, - } - return pipelineCfg, nil -} - -func getMetricPipelineName(clusterID, processorName string) string{ - return fmt.Sprintf("collect_%s_%s", clusterID, processorName) -} - - -func LoadAgentsFromES(clusterID string) ([]model.Instance, error) { - q := orm.Query{ - Size: 1000, - } - if clusterID != "" { - q.Conds = orm.And(orm.Eq("id", clusterID)) - } - err, result := orm.Search(model.Instance{}, &q) - if err != nil { - return nil, fmt.Errorf("query agent error: %w", err) - } - - if len(result.Result) > 0 { - var agents = make([]model.Instance, 0, len(result.Result)) - for _, row := range result.Result { - ag := model.Instance{} - bytes := util.MustToJSONBytes(row) - err = util.FromJSONBytes(bytes, &ag) - if err != nil { - log.Errorf("got unexpected agent: %s, error: %v", string(bytes), err) - continue - } - agents = append(agents, ag) - } - return agents, nil - } - return nil, nil -} - -func GetLatestOnlineAgentIDs(agentIds []string, lastSeconds int) (map[string]struct{}, error) { - q := orm.Query{ - WildcardIndex: true, - } - mustQ := []util.MapStr{ - { - "term": util.MapStr{ - "metadata.name": util.MapStr{ - "value": "agent", - }, - }, - }, - { - "term": util.MapStr{ - "metadata.category": util.MapStr{ - "value": "instance", - }, - }, - }, - } - if len(agentIds) > 0 { - mustQ = append(mustQ, util.MapStr{ - "terms": util.MapStr{ - "agent.id": agentIds, - }, - }) - } - queryDSL := util.MapStr{ - "sort": []util.MapStr{ - { - "timestamp": util.MapStr{ - "order": "desc", - }, - }, - }, - "collapse": util.MapStr{ - "field": "agent.id", - }, - "query": util.MapStr{ - "bool": util.MapStr{ - "filter": []util.MapStr{ - { - "range": util.MapStr{ - "timestamp": util.MapStr{ - "gte": fmt.Sprintf("now-%ds", lastSeconds), - }, - }, - }, - }, - "must": mustQ, - }, - }, - } - if len(agentIds) == 0 { - queryDSL["size"] = 2000 - }else{ - queryDSL["size"] = len(agentIds) - } - q.RawQuery = util.MustToJSONBytes(queryDSL) - err, result := orm.Search(event.Event{}, &q) - if err != nil { - return nil, fmt.Errorf("query agent instance metric error: %w", err) - } - agentIDs := map[string]struct{}{} - if len(result.Result) > 0 { - searchRes := elastic.SearchResponse{} - err = util.FromJSONBytes(result.Raw, &searchRes) - if err != nil { - return nil, err - } - agentIDKeyPath := []string{"agent", "id"} - for _, hit := range searchRes.Hits.Hits { - agentID, _ := util.GetMapValueByKeys(agentIDKeyPath, hit.Source) - if v, ok := agentID.(string); ok { - agentIDs[v] = struct{}{} - } - } - } - return agentIDs, nil -} - -func GetAgentIngestConfig() (string, *model.BasicAuth, error) { - agCfg := GetAgentConfig() - var ( - endpoint string - ok bool - ) - emptyIngestClusterEndpoint := false - if agCfg.Setup.IngestClusterEndpoint == nil { - emptyIngestClusterEndpoint = true - } - if endpoint, ok = agCfg.Setup.IngestClusterEndpoint.(string);ok { - if endpoint = strings.TrimSpace(endpoint); endpoint == "" { - emptyIngestClusterEndpoint = true - } - } - if emptyIngestClusterEndpoint { - cfg := elastic.GetConfig(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) - endpoint = cfg.GetAnyEndpoint() - } - - var ( - basicAuth model.BasicAuth - ) - if agCfg.Setup.IngestClusterCredentialID != "" { - cred := credential.Credential{} - cred.ID = agCfg.Setup.IngestClusterCredentialID - _, err := orm.Get(&cred) - if err != nil { - return "", nil, fmt.Errorf("query credential [%s] error: %w", cred.ID, err) - } - info, err := cred.Decode() - if err != nil { - return "", nil, fmt.Errorf("decode credential [%s] error: %w", cred.ID, err) - } - if basicAuth, ok = info.(model.BasicAuth); !ok { - log.Debug("invalid credential: ", cred) - } - }else{ - cfg := elastic.GetConfig(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) - basicAuth = *cfg.BasicAuth - } - tpl := `configs.template: - - name: "ingest" - path: ./config/ingest_config.tpl - variable: - INGEST_CLUSTER_ID: "default_ingest_cluster" - INGEST_CLUSTER_ENDPOINT: ["%s"] - INGEST_CLUSTER_USERNAME: "%s" -` - tpl = fmt.Sprintf(tpl, endpoint, basicAuth.Username) - return tpl, &basicAuth, nil -} \ No newline at end of file diff --git a/modules/agent/model/config.go b/modules/agent/model/config.go index d731a3f8..d20526b3 100644 --- a/modules/agent/model/config.go +++ b/modules/agent/model/config.go @@ -6,9 +6,6 @@ package model type AgentConfig struct { Enabled bool `config:"enabled"` - StateManager struct { - Enabled bool `config:"enabled"` - } `config:"state_manager"` Setup *SetupConfig `config:"setup"` } @@ -18,7 +15,5 @@ type SetupConfig struct { CACertFile string `config:"ca_cert"` CAKeyFile string `config:"ca_key"` ConsoleEndpoint string `config:"console_endpoint"` - IngestClusterEndpoint interface{} `config:"ingest_cluster_endpoint"` - IngestClusterCredentialID string `config:"ingest_cluster_credential_id"` Port string `config:"port"` } diff --git a/modules/agent/model/task.go b/modules/agent/model/task.go deleted file mode 100644 index eed98ee4..00000000 --- a/modules/agent/model/task.go +++ /dev/null @@ -1,46 +0,0 @@ -/* Copyright © INFINI Ltd. All rights reserved. - * Web: https://infinilabs.com - * Email: hello#infini.ltd */ - -package model - -import ( - "infini.sh/framework/core/elastic" - "infini.sh/framework/core/util" -) - -type TaskSetting struct { - ClusterHealth *ClusterHealthTask `json:"cluster_health,omitempty"` - ClusterStats *ClusterStatsTask `json:"cluster_stats,omitempty"` - IndexStats *IndexStatsTask `json:"index_stats,omitempty"` - NodeStats *NodeStatsTask `json:"node_stats,omitempty"` - Logs *LogsTask `json:"logs,omitempty"` -} - -type ClusterHealthTask struct { - Enabled bool `json:"enabled"` -} - -type ClusterStatsTask struct { - Enabled bool `json:"enabled"` -} - -type IndexStatsTask struct { - Enabled bool `json:"enabled"` -} - -type NodeStatsTask struct { - Enabled bool `json:"enabled"` - NodeIDs []string `json:"node_ids,omitempty"` -} - -type LogsTask struct { - Enabled bool `json:"enabled"` - LogsPath string `json:"logs_path"` -} - -type ParseAgentSettingsResult struct { - ClusterConfigs []elastic.ElasticsearchConfig - Pipelines []util.MapStr - ToDeletePipelineNames []string -} From add547e78f6def957aa3ce7c6b2afd44d1eea3f0 Mon Sep 17 00:00:00 2001 From: medcl Date: Fri, 20 Oct 2023 18:52:51 +0800 Subject: [PATCH 12/36] refactoring --- modules/agent/api/elasticsearch.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/agent/api/elasticsearch.go b/modules/agent/api/elasticsearch.go index 1926428a..101492b6 100644 --- a/modules/agent/api/elasticsearch.go +++ b/modules/agent/api/elasticsearch.go @@ -5,7 +5,6 @@ package api import ( - "bytes" "context" "errors" "fmt" @@ -16,8 +15,7 @@ import ( "infini.sh/framework/core/model" "infini.sh/framework/core/orm" "infini.sh/framework/core/util" - common2 "infini.sh/framework/modules/elastic/common" - "infini.sh/framework/plugins/managed/common" + "infini.sh/framework/modules/elastic/common" "infini.sh/framework/plugins/managed/server" "net/http" "net/url" @@ -347,7 +345,7 @@ func (h *APIHandler) discoveryESNodesInfo(w http.ResponseWriter, req *http.Reque meta := elastic.GetMetadata(clusterID) if meta != nil { if meta.Config.AgentCredentialID != "" { - auth, err := common2.GetAgentBasicAuth(meta.Config) + auth, err := common.GetAgentBasicAuth(meta.Config) if err != nil { panic(err) } From 7b84271ab35e882dc003382712890fe01d1b4617 Mon Sep 17 00:00:00 2001 From: medcl Date: Fri, 20 Oct 2023 20:31:23 +0800 Subject: [PATCH 13/36] refactoring agent proxy --- modules/agent/api/elasticsearch.go | 73 +++++++----------------------- 1 file changed, 17 insertions(+), 56 deletions(-) diff --git a/modules/agent/api/elasticsearch.go b/modules/agent/api/elasticsearch.go index 101492b6..d7ca9ffa 100644 --- a/modules/agent/api/elasticsearch.go +++ b/modules/agent/api/elasticsearch.go @@ -6,20 +6,16 @@ package api import ( "context" - "errors" "fmt" log "github.com/cihub/seelog" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" - "infini.sh/framework/core/global" "infini.sh/framework/core/model" "infini.sh/framework/core/orm" "infini.sh/framework/core/util" "infini.sh/framework/modules/elastic/common" "infini.sh/framework/plugins/managed/server" "net/http" - "net/url" - "sync" "time" ) @@ -113,7 +109,7 @@ func GetElasticsearchNodesViaAgent(ctx context.Context, instance *model.Instance } obj := elastic.DiscoveryResult{} - err := doRequest(instance, req, &obj) + _,err := server.ProxyAgentRequest(instance.GetEndpoint(), req, &obj) if err != nil { return nil, err } @@ -149,7 +145,7 @@ func GetElasticLogFiles(ctx context.Context, instance *model.Instance, logsPath } resBody := map[string]interface{}{} - err := doRequest(instance, req, &resBody) + _,err := server.ProxyAgentRequest(instance.GetEndpoint(), req, &resBody) if err != nil { return nil, err } @@ -168,7 +164,7 @@ func GetElasticLogFileContent(ctx context.Context, instance *model.Instance, bod Body: util.MustToJSONBytes(body), } resBody := map[string]interface{}{} - err := doRequest(instance, req, &resBody) + _,err := server.ProxyAgentRequest(instance.GetEndpoint(), req, &resBody) if err != nil { return nil, err } @@ -397,17 +393,23 @@ func (h *APIHandler) discoveryESNodesInfo(w http.ResponseWriter, req *http.Reque func (h *APIHandler) getESNodeInfoViaProxy(host string, schema string, auth *model.BasicAuth, instance model.Instance) (success, tryAgain bool, info *elastic.LocalNodeInfo) { esConfig := elastic.ElasticsearchConfig{Host: host, Schema: schema, BasicAuth: auth} body := util.MustToJSONBytes(esConfig) - res, err := server.ProxyRequestToRuntimeInstance(instance.GetEndpoint(), "POST", "/elasticsearch/node/_info", - body, int64(len(body)), auth) - if err != nil { - panic(err) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + req := &util.Request{ + Method: http.MethodPost, + Path: "/elasticsearch/node/_info", + Context: ctx, + Body: body, + } + if auth != nil { + req.SetBasicAuth(auth.Username, auth.Password) } - if global.Env().IsDebug { - if res != nil { // && res.StatusCode==http.StatusOK - log.Debug(string(res.Body)) - } + obj := elastic.LocalNodeInfo{} + res,err := server.ProxyAgentRequest(instance.GetEndpoint(), req, &obj) + if err != nil { + panic(err) } if res != nil && res.StatusCode == http.StatusForbidden { @@ -539,44 +541,3 @@ func (h *APIHandler) enrollESNode(w http.ResponseWriter, req *http.Request, ps h h.WriteAckOKJSON(w) } - -var mTLSClient *http.Client //TODO get mTLSClient -var initOnce = sync.Once{} - -func doRequest(instance *model.Instance, req *util.Request, obj interface{}) error { - var err error - var res *util.Result - - initOnce.Do(func() { - if global.Env().SystemConfig.Configs.TLSConfig.TLSEnabled && global.Env().SystemConfig.Configs.TLSConfig.TLSCAFile != "" { - - //init client - hClient, err := util.NewMTLSClient( - global.Env().SystemConfig.Configs.TLSConfig.TLSCAFile, - global.Env().SystemConfig.Configs.TLSConfig.TLSCertFile, - global.Env().SystemConfig.Configs.TLSConfig.TLSKeyFile) - if err != nil { - panic(err) - } - mTLSClient = hClient - } - }) - - req.Url, err = url.JoinPath(instance.GetEndpoint(), req.Path) - res, err = util.ExecuteRequestWithCatchFlag(mTLSClient, req, true) - if err != nil || res.StatusCode != 200 { - body := "" - if res != nil { - body = string(res.Body) - } - return errors.New(fmt.Sprintf("request error: %v, %v", err, body)) - } - - if res != nil { - if res.Body != nil { - return util.FromJSONBytes(res.Body, obj) - } - } - - return nil -} From 0c4edd45415d9ef85d24e5f0996883307fc850b9 Mon Sep 17 00:00:00 2001 From: medcl Date: Fri, 20 Oct 2023 23:55:26 +0800 Subject: [PATCH 14/36] fix enroll api, should check agent credential before enroll --- config/install_agent.tpl | 2 +- modules/agent/api/elasticsearch.go | 74 +++++++++++++++++++++--------- modules/agent/api/remote_config.go | 18 +++++--- 3 files changed, 66 insertions(+), 28 deletions(-) diff --git a/config/install_agent.tpl b/config/install_agent.tpl index 1b0881d5..610eaefe 100644 --- a/config/install_agent.tpl +++ b/config/install_agent.tpl @@ -267,7 +267,7 @@ configs: #for managed client's setting managed: true # managed by remote servers panic_on_config_error: false #ignore config error - interval: "1s" + interval: "10s" servers: # config servers - "http://localhost:9000" max_backup_files: 5 diff --git a/modules/agent/api/elasticsearch.go b/modules/agent/api/elasticsearch.go index d7ca9ffa..13bf827d 100644 --- a/modules/agent/api/elasticsearch.go +++ b/modules/agent/api/elasticsearch.go @@ -10,6 +10,7 @@ import ( log "github.com/cihub/seelog" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" + "infini.sh/framework/core/global" "infini.sh/framework/core/model" "infini.sh/framework/core/orm" "infini.sh/framework/core/util" @@ -109,7 +110,7 @@ func GetElasticsearchNodesViaAgent(ctx context.Context, instance *model.Instance } obj := elastic.DiscoveryResult{} - _,err := server.ProxyAgentRequest(instance.GetEndpoint(), req, &obj) + _, err := server.ProxyAgentRequest(instance.GetEndpoint(), req, &obj) if err != nil { return nil, err } @@ -145,7 +146,7 @@ func GetElasticLogFiles(ctx context.Context, instance *model.Instance, logsPath } resBody := map[string]interface{}{} - _,err := server.ProxyAgentRequest(instance.GetEndpoint(), req, &resBody) + _, err := server.ProxyAgentRequest(instance.GetEndpoint(), req, &resBody) if err != nil { return nil, err } @@ -164,7 +165,7 @@ func GetElasticLogFileContent(ctx context.Context, instance *model.Instance, bod Body: util.MustToJSONBytes(body), } resBody := map[string]interface{}{} - _,err := server.ProxyAgentRequest(instance.GetEndpoint(), req, &resBody) + _, err := server.ProxyAgentRequest(instance.GetEndpoint(), req, &resBody) if err != nil { return nil, err } @@ -354,10 +355,10 @@ func (h *APIHandler) discoveryESNodesInfo(w http.ResponseWriter, req *http.Reque ip = fmt.Sprintf("[%s]", v.IP) } nodeHost := fmt.Sprintf("%s:%d", ip, v.Port) - success, tryAgain, nodeInfo := h.getESNodeInfoViaProxy(nodeHost, "http", auth, instance) + success, tryAgain, nodeInfo := h.getESNodeInfoViaProxy(nodeHost, "http", auth, &instance) if !success && tryAgain { //try https again - success, tryAgain, nodeInfo = h.getESNodeInfoViaProxy(nodeHost, "https", auth, instance) + success, tryAgain, nodeInfo = h.getESNodeInfoViaProxy(nodeHost, "https", auth, &instance) } if success { log.Error("connect to es node success:", nodeHost) @@ -390,26 +391,32 @@ func (h *APIHandler) discoveryESNodesInfo(w http.ResponseWriter, req *http.Reque h.WriteJSON(w, nodes, http.StatusOK) } -func (h *APIHandler) getESNodeInfoViaProxy(host string, schema string, auth *model.BasicAuth, instance model.Instance) (success, tryAgain bool, info *elastic.LocalNodeInfo) { - esConfig := elastic.ElasticsearchConfig{Host: host, Schema: schema, BasicAuth: auth} - body := util.MustToJSONBytes(esConfig) +func (h *APIHandler) getESNodeInfoViaProxy(esHost string, esSchema string, auth *model.BasicAuth, instance *model.Instance) (success, tryAgain bool, info *elastic.LocalNodeInfo) { + esConfig := elastic.ElasticsearchConfig{Host: esHost, Schema: esSchema, BasicAuth: auth} + return h.getESNodeInfoViaProxyWithConfig(&esConfig, auth, instance) +} +func (h *APIHandler) getESNodeInfoViaProxyWithConfig(cfg *elastic.ElasticsearchConfig, auth *model.BasicAuth, instance *model.Instance) (success, tryAgain bool, info *elastic.LocalNodeInfo) { + body := util.MustToJSONBytes(cfg) ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() req := &util.Request{ Method: http.MethodPost, Path: "/elasticsearch/node/_info", Context: ctx, - Body: body, + Body: body, } if auth != nil { req.SetBasicAuth(auth.Username, auth.Password) } - obj := elastic.LocalNodeInfo{} - res,err := server.ProxyAgentRequest(instance.GetEndpoint(), req, &obj) + obj := elastic.LocalNodeInfo{} + res, err := server.ProxyAgentRequest(instance.GetEndpoint(), req, &obj) if err != nil { - panic(err) + if global.Env().IsDebug { + log.Error(err) + } + return false, true, nil } if res != nil && res.StatusCode == http.StatusForbidden { @@ -521,23 +528,48 @@ func (h *APIHandler) enrollESNode(w http.ResponseWriter, req *http.Request, ps h //agent id instID := ps.MustGetParameter("instance_id") + exists, instance, err := server.GetRuntimeInstanceByID(instID) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + if !exists { + h.WriteError(w, "instance not found", http.StatusInternalServerError) + } + //node id and cluster id item := BindingItem{} - err := h.DecodeJSON(req, &item) + err = h.DecodeJSON(req, &item) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) return } - //update node's setting - settings := NewNodeAgentSettings(instID, &item) - err = orm.Update(&orm.Context{ - Refresh: "wait_for", - }, settings) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) + //check if the cluster's agent credential is valid + meta := elastic.GetMetadata(item.ClusterID) + if meta == nil { + h.WriteError(w, "cluster not found", http.StatusInternalServerError) return } - h.WriteAckOKJSON(w) + //use agent credential to access the node + meta.Config.BasicAuth, _ = common.GetAgentBasicAuth(meta.Config) + + success, _, _ := h.getESNodeInfoViaProxyWithConfig(meta.Config, meta.Config.BasicAuth, instance) + + if success { + //update node's setting + settings := NewNodeAgentSettings(instID, &item) + err = orm.Update(&orm.Context{ + Refresh: "wait_for", + }, settings) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.WriteAckOKJSON(w) + } else { + h.WriteError(w, "failed to access this node", http.StatusInternalServerError) + } } diff --git a/modules/agent/api/remote_config.go b/modules/agent/api/remote_config.go index e30fd7c2..7660c2ae 100644 --- a/modules/agent/api/remote_config.go +++ b/modules/agent/api/remote_config.go @@ -112,8 +112,9 @@ func dynamicAgentConfigProvider(instance model.Instance) []*common.ConfigFile { func getAgentIngestConfigs(items map[string]BindingItem) string { - buffer := bytes.NewBuffer([]byte("configs.template:\n ")) + buffer := bytes.NewBuffer([]byte("configs.template: ")) + var latestVersion int64 for _, v := range items { if v.ClusterID == "" { @@ -144,7 +145,11 @@ func getAgentIngestConfigs(items map[string]BindingItem) string { } } - buffer.Write([]byte(fmt.Sprintf("- name: \"%v\"\n path: ./config/task_config.tpl\n "+ + if v.Updated > latestVersion { + latestVersion = v.Updated + } + + buffer.Write([]byte(fmt.Sprintf("\n - name: \"%v\"\n path: ./config/task_config.tpl\n "+ "variable:\n "+ "CLUSTER_ID: %v\n "+ "CLUSTER_ENDPOINT: [\"%v\"]\n "+ @@ -152,12 +157,13 @@ func getAgentIngestConfigs(items map[string]BindingItem) string { "CLUSTER_PASSWORD: \"%v\"\n "+ "CLUSTER_LEVEL_TASKS_ENABLED: %v\n "+ "NODE_LEVEL_TASKS_ENABLED: %v\n "+ - "NODE_LOGS_PATH: \"%v\"\n\n\n"+ - "#MANAGED_CONFIG_VERSION: %v\n"+ - "#MANAGED: true", - v.NodeUUID, v.ClusterID, clusterEndPoint, username, password, clusterLevelEnabled, nodeLevelEnabled, v.PathLogs, v.Updated))) + "NODE_LOGS_PATH: \"%v\"\n\n\n", v.NodeUUID, v.ClusterID, clusterEndPoint, username, password, clusterLevelEnabled, nodeLevelEnabled, v.PathLogs))) } + + buffer.WriteString("\n") + buffer.WriteString(fmt.Sprintf("#MANAGED_CONFIG_VERSION: %v\n#MANAGED: true\n",latestVersion)) + //password: $[[keystore.$[[CLUSTER_ID]]_password]] return buffer.String() From a6656842bf2c6d755cef4fe930ec385e1f3a0c91 Mon Sep 17 00:00:00 2001 From: medcl Date: Fri, 20 Oct 2023 23:55:38 +0800 Subject: [PATCH 15/36] add default template --- config/initialization.tpl | 254 +++++++++++++++++++++++++++++++++++++- 1 file changed, 253 insertions(+), 1 deletion(-) diff --git a/config/initialization.tpl b/config/initialization.tpl index 27953449..1f710465 100644 --- a/config/initialization.tpl +++ b/config/initialization.tpl @@ -5709,6 +5709,258 @@ POST $[[INDEX_PREFIX]]layout/_doc/cicmhbt3q95ich72lrvg "is_fixed": true } -GET / +#agent + + +POST .infini_configs/_doc/task_config_tpl +{ + "id": "task_config_tpl", + "updated": "2023-10-19T14:49:56.768754+08:00", + "metadata": { + "category": "app_settings", + "name": "agent", + "labels": { + "instance": "_all" + } + }, + "payload": { + "name": "task_config.tpl", + "location": "task_config.tpl", + "content": """env: + CLUSTER_PASSWORD: $[[keystore.$[[CLUSTER_ID]]_password]] + +elasticsearch: + - id: $[[CLUSTER_ID]] + name: $[[CLUSTER_ID]] + enabled: true + endpoints: $[[CLUSTER_ENDPOINT]] + discovery: + enabled: false + basic_auth: + username: $[[CLUSTER_USERNAME]] + password: $[[CLUSTER_PASSWORD]] + +pipeline: +#clsuter level metrics +- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + keep_running: true + singleton: true + name: collect_$[[CLUSTER_ID]]_es_cluster_stats + retry_delay_in_ms: 10000 + processor: + - es_cluster_stats: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + when: + cluster_available: ["$[[CLUSTER_ID]]"] + +- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + keep_running: true + singleton: true + name: collect_$[[CLUSTER_ID]]_es_index_stats + retry_delay_in_ms: 10000 + processor: + - es_index_stats: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + when: + cluster_available: ["$[[CLUSTER_ID]]"] + +- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + keep_running: true + singleton: true + name: collect_$[[CLUSTER_ID]]_es_cluster_health + retry_delay_in_ms: 10000 + processor: + - es_cluster_health: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + when: + cluster_available: ["$[[CLUSTER_ID]]"] + +#node level metrics +- auto_start: $[[NODE_LEVEL_TASKS_ENABLED]] + enabled: $[[NODE_LEVEL_TASKS_ENABLED]] + keep_running: true + name: collect_$[[CLUSTER_ID]]_es_node_stats + retry_delay_in_ms: 10000 + processor: + - es_node_stats: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + when: + cluster_available: ["$[[CLUSTER_ID]]"] + +#node logs +- auto_start: $[[NODE_LEVEL_TASKS_ENABLED]] + enabled: $[[NODE_LEVEL_TASKS_ENABLED]] + keep_running: true + name: collect_$[[CLUSTER_ID]]_es_logs + retry_delay_in_ms: 10000 + processor: + - es_logs_processor: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + logs_path: $[[NODE_LOGS_PATH]] + queue_name: logs + when: + cluster_available: ["$[[CLUSTER_ID]]"] +""", + "version": 1 + } +} + +POST .infini_configs/_doc/ingest_config_tpl +{ + "id": "ingest_config_tpl", + "updated": "2023-10-19T14:49:56.768754+08:00", + "metadata": { + "category": "app_settings", + "name": "agent", + "labels": { + "instance": "_all" + } + }, + "payload": { + "name": "ingest_config.tpl", + "location": "ingest_config.tpl", + "content": """elasticsearch: + - name: $[[INGEST_CLUSTER_ID]] + enabled: true + endpoints: $[[INGEST_CLUSTER_ENDPOINT]] + discovery: + enabled: false + basic_auth: + username: $[[INGEST_CLUSTER_USERNAME]] + password: $[[keystore.ingest_cluster_password]] + +metrics: + enabled: true + queue: metrics + network: + enabled: true + summary: true + details: true + memory: + metrics: + - swap + - memory + disk: + metrics: + - iops + - usage + cpu: + metrics: + - idle + - system + - user + - iowait + - load + instance: + enabled: true + +elastic: + availability_check: + enabled: false + +pipeline: + - name: merge_logs + auto_start: true + keep_running: true + processor: + - indexing_merge: + elasticsearch: "$[[INGEST_CLUSTER_ID]]" + index_name: ".infini_logs" + type_name: "_doc" + input_queue: "logs" + idle_timeout_in_seconds: 10 + output_queue: + name: "merged_requests" + worker_size: 1 + bulk_size_in_mb: 5 + - name: merge_metrics + auto_start: true + keep_running: true + processor: + - indexing_merge: + elasticsearch: "$[[INGEST_CLUSTER_ID]]" + index_name: ".infini_metrics" + type_name: "_doc" + input_queue: "metrics" + output_queue: + name: "merged_requests" + worker_size: 1 + bulk_size_in_mb: 5 + - name: ingest_merged_requests + enabled: true + auto_start: true + keep_running: true + processor: + - bulk_indexing: + max_worker_size: 1 + verbose_bulk_result: false + bulk: + batch_size_in_mb: 5 + batch_size_in_docs: 5000 + max_retry_times: 0 + invalid_queue: "" + response_handle: + include_index_stats: false + include_action_stats: false + output_bulk_stats: false + include_error_details: true + save_error_results: true + save_success_results: false + save_busy_results: false + consumer: + fetch_max_messages: 5 + queues: + type: indexing_merge + when: + cluster_available: ["$[[INGEST_CLUSTER_ID]]"] +""", + "version": 1 + } +} + + +POST .infini_configs/_doc/system_ingest_config_yml +{ + "id": "system_ingest_config_yml", + "updated": "2023-10-19T14:49:56.768754+08:00", + "metadata": { + "category": "app_settings", + "name": "agent", + "labels": { + "instance": "_all" + } + }, + "payload": { + "name": "system_ingest_config.yml", + "location": "system_ingest_config.yml", + "content": """configs.template: + - name: "default_ingest_config" + path: ./config/ingest_config.tpl + variable: + INGEST_CLUSTER_ID: infini_default_ingest_cluster + INGEST_CLUSTER_ENDPOINT: [ "https://localhost:9200" ] + INGEST_CLUSTER_USERNAME: "admin" + CLUSTER_VER: "1.6.0" + CLUSTER_DISTRIBUTION: "easysearch" + INDEX_PREFIX: ".infini_" + +""", + "version": 1 + } +} + From cb9cc545e5642e2428641069f9537303bf0e5ee3 Mon Sep 17 00:00:00 2001 From: medcl Date: Sat, 21 Oct 2023 00:25:32 +0800 Subject: [PATCH 16/36] fix incorrect variable name --- plugin/setup/setup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/setup/setup.go b/plugin/setup/setup.go index a27cad77..565f4ecc 100644 --- a/plugin/setup/setup.go +++ b/plugin/setup/setup.go @@ -568,7 +568,7 @@ func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps http //save to local file file := path.Join(global.Env().GetConfigDir(), "system_config.yml") _, err = util.FilePutContent(file, fmt.Sprintf("configs.template:\n - name: \"system\"\n path: ./config/system_config.tpl\n variable:\n "+ - "CLUSTER_ID: %v\n CLUSTER_ENDPINT: \"%v\"\n "+ + "CLUSTER_ID: %v\n CLUSTER_ENDPOINT: \"%v\"\n "+ "CLUSTER_USER: \"%v\"\n CLUSTER_VER: \"%v\"\n CLUSTER_DISTRIBUTION: \"%v\"\n INDEX_PREFIX: \"%v\"", GlobalSystemElasticsearchID, cfg.GetAnyEndpoint(), cfg.BasicAuth.Username, cfg.Version, cfg.Distribution, cfg1.IndexPrefix)) if err != nil { From 9568e800caa66fa8d028532c575a99cc37df0ec7 Mon Sep 17 00:00:00 2001 From: medcl Date: Sat, 21 Oct 2023 13:54:10 +0800 Subject: [PATCH 17/36] separate init files for agent --- config/init_agent_config.tpl | 254 ++++++++++++++++++++++++++++++++++ config/initialization.tpl | 258 +---------------------------------- 2 files changed, 255 insertions(+), 257 deletions(-) create mode 100644 config/init_agent_config.tpl diff --git a/config/init_agent_config.tpl b/config/init_agent_config.tpl new file mode 100644 index 00000000..70f07ddb --- /dev/null +++ b/config/init_agent_config.tpl @@ -0,0 +1,254 @@ +#agent + +POST .infini_configs/_doc/task_config_tpl +{ + "id": "task_config_tpl", + "updated": "2023-10-19T14:49:56.768754+08:00", + "metadata": { + "category": "app_settings", + "name": "agent", + "labels": { + "instance": "_all" + } + }, + "payload": { + "name": "task_config.tpl", + "location": "task_config.tpl", + "content": """env: + CLUSTER_PASSWORD: $[[keystore.$[[CLUSTER_ID]]_password]] + +elasticsearch: + - id: $[[CLUSTER_ID]] + name: $[[CLUSTER_ID]] + enabled: true + endpoints: $[[CLUSTER_ENDPOINT]] + discovery: + enabled: false + basic_auth: + username: $[[CLUSTER_USERNAME]] + password: $[[CLUSTER_PASSWORD]] + +pipeline: +#clsuter level metrics +- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + keep_running: true + singleton: true + name: collect_$[[CLUSTER_ID]]_es_cluster_stats + retry_delay_in_ms: 10000 + processor: + - es_cluster_stats: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + when: + cluster_available: ["$[[CLUSTER_ID]]"] + +- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + keep_running: true + singleton: true + name: collect_$[[CLUSTER_ID]]_es_index_stats + retry_delay_in_ms: 10000 + processor: + - es_index_stats: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + when: + cluster_available: ["$[[CLUSTER_ID]]"] + +- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + keep_running: true + singleton: true + name: collect_$[[CLUSTER_ID]]_es_cluster_health + retry_delay_in_ms: 10000 + processor: + - es_cluster_health: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + when: + cluster_available: ["$[[CLUSTER_ID]]"] + +#node level metrics +- auto_start: $[[NODE_LEVEL_TASKS_ENABLED]] + enabled: $[[NODE_LEVEL_TASKS_ENABLED]] + keep_running: true + name: collect_$[[CLUSTER_ID]]_es_node_stats + retry_delay_in_ms: 10000 + processor: + - es_node_stats: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + when: + cluster_available: ["$[[CLUSTER_ID]]"] + +#node logs +- auto_start: $[[NODE_LEVEL_TASKS_ENABLED]] + enabled: $[[NODE_LEVEL_TASKS_ENABLED]] + keep_running: true + name: collect_$[[CLUSTER_ID]]_es_logs + retry_delay_in_ms: 10000 + processor: + - es_logs_processor: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + logs_path: $[[NODE_LOGS_PATH]] + queue_name: logs + when: + cluster_available: ["$[[CLUSTER_ID]]"] +""", + "version": 1 + } +} + +POST .infini_configs/_doc/ingest_config_tpl +{ + "id": "ingest_config_tpl", + "updated": "2023-10-19T14:49:56.768754+08:00", + "metadata": { + "category": "app_settings", + "name": "agent", + "labels": { + "instance": "_all" + } + }, + "payload": { + "name": "ingest_config.tpl", + "location": "ingest_config.tpl", + "content": """elasticsearch: + - name: $[[INGEST_CLUSTER_ID]] + enabled: true + endpoints: $[[INGEST_CLUSTER_ENDPOINT]] + discovery: + enabled: false + basic_auth: + username: $[[INGEST_CLUSTER_USERNAME]] + password: $[[INGEST_CLUSTER_PASSWORD]] + +metrics: + enabled: true + queue: metrics + network: + enabled: true + summary: true + details: true + memory: + metrics: + - swap + - memory + disk: + metrics: + - iops + - usage + cpu: + metrics: + - idle + - system + - user + - iowait + - load + instance: + enabled: true + +elastic: + availability_check: + enabled: false + +pipeline: + - name: merge_logs + auto_start: true + keep_running: true + processor: + - indexing_merge: + elasticsearch: "$[[INGEST_CLUSTER_ID]]" + index_name: ".infini_logs" + type_name: "_doc" + input_queue: "logs" + idle_timeout_in_seconds: 10 + output_queue: + name: "merged_requests" + worker_size: 1 + bulk_size_in_mb: 5 + - name: merge_metrics + auto_start: true + keep_running: true + processor: + - indexing_merge: + elasticsearch: "$[[INGEST_CLUSTER_ID]]" + index_name: ".infini_metrics" + type_name: "_doc" + input_queue: "metrics" + output_queue: + name: "merged_requests" + worker_size: 1 + bulk_size_in_mb: 5 + - name: ingest_merged_requests + enabled: true + auto_start: true + keep_running: true + processor: + - bulk_indexing: + max_worker_size: 1 + verbose_bulk_result: false + bulk: + batch_size_in_mb: 5 + batch_size_in_docs: 5000 + max_retry_times: 0 + invalid_queue: "" + response_handle: + include_index_stats: false + include_action_stats: false + output_bulk_stats: false + include_error_details: true + save_error_results: true + save_success_results: false + save_busy_results: false + consumer: + fetch_max_messages: 5 + queues: + type: indexing_merge + when: + cluster_available: ["$[[INGEST_CLUSTER_ID]]"] +""", + "version": 1 + } +} + + +#TODO, need to replace cleartext password to keystore, and ingest endpoint +POST .infini_configs/_doc/system_ingest_config_yml +{ + "id": "system_ingest_config_yml", + "updated": "2023-10-18T14:49:56.768754+08:00", + "metadata": { + "category": "app_settings", + "name": "agent", + "labels": { + "instance": "_all" + } + }, + "payload": { + "name": "system_ingest_config.yml", + "location": "system_ingest_config.yml", + "content": """configs.template: + - name: "default_ingest_config" + path: ./config/ingest_config.tpl + variable: + INGEST_CLUSTER_ID: infini_default_ingest_cluster + INGEST_CLUSTER_ENDPOINT: [ "http://10.0.0.3:7102" ] + INGEST_CLUSTER_USERNAME: "admin" + INGEST_CLUSTER_PASSWORD: "admin" + CLUSTER_VER: "1.6.0" + CLUSTER_DISTRIBUTION: "easysearch" + INDEX_PREFIX: ".infini_" + +""", + "version": 2 + } +} + diff --git a/config/initialization.tpl b/config/initialization.tpl index 1f710465..1d795676 100644 --- a/config/initialization.tpl +++ b/config/initialization.tpl @@ -5707,260 +5707,4 @@ POST $[[INDEX_PREFIX]]layout/_doc/cicmhbt3q95ich72lrvg }, "type": "workspace", "is_fixed": true -} - - - -#agent - - -POST .infini_configs/_doc/task_config_tpl -{ - "id": "task_config_tpl", - "updated": "2023-10-19T14:49:56.768754+08:00", - "metadata": { - "category": "app_settings", - "name": "agent", - "labels": { - "instance": "_all" - } - }, - "payload": { - "name": "task_config.tpl", - "location": "task_config.tpl", - "content": """env: - CLUSTER_PASSWORD: $[[keystore.$[[CLUSTER_ID]]_password]] - -elasticsearch: - - id: $[[CLUSTER_ID]] - name: $[[CLUSTER_ID]] - enabled: true - endpoints: $[[CLUSTER_ENDPOINT]] - discovery: - enabled: false - basic_auth: - username: $[[CLUSTER_USERNAME]] - password: $[[CLUSTER_PASSWORD]] - -pipeline: -#clsuter level metrics -- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] - enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] - keep_running: true - singleton: true - name: collect_$[[CLUSTER_ID]]_es_cluster_stats - retry_delay_in_ms: 10000 - processor: - - es_cluster_stats: - elasticsearch: $[[CLUSTER_ID]] - labels: - cluster_id: $[[CLUSTER_ID]] - when: - cluster_available: ["$[[CLUSTER_ID]]"] - -- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] - enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] - keep_running: true - singleton: true - name: collect_$[[CLUSTER_ID]]_es_index_stats - retry_delay_in_ms: 10000 - processor: - - es_index_stats: - elasticsearch: $[[CLUSTER_ID]] - labels: - cluster_id: $[[CLUSTER_ID]] - when: - cluster_available: ["$[[CLUSTER_ID]]"] - -- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] - enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] - keep_running: true - singleton: true - name: collect_$[[CLUSTER_ID]]_es_cluster_health - retry_delay_in_ms: 10000 - processor: - - es_cluster_health: - elasticsearch: $[[CLUSTER_ID]] - labels: - cluster_id: $[[CLUSTER_ID]] - when: - cluster_available: ["$[[CLUSTER_ID]]"] - -#node level metrics -- auto_start: $[[NODE_LEVEL_TASKS_ENABLED]] - enabled: $[[NODE_LEVEL_TASKS_ENABLED]] - keep_running: true - name: collect_$[[CLUSTER_ID]]_es_node_stats - retry_delay_in_ms: 10000 - processor: - - es_node_stats: - elasticsearch: $[[CLUSTER_ID]] - labels: - cluster_id: $[[CLUSTER_ID]] - when: - cluster_available: ["$[[CLUSTER_ID]]"] - -#node logs -- auto_start: $[[NODE_LEVEL_TASKS_ENABLED]] - enabled: $[[NODE_LEVEL_TASKS_ENABLED]] - keep_running: true - name: collect_$[[CLUSTER_ID]]_es_logs - retry_delay_in_ms: 10000 - processor: - - es_logs_processor: - elasticsearch: $[[CLUSTER_ID]] - labels: - cluster_id: $[[CLUSTER_ID]] - logs_path: $[[NODE_LOGS_PATH]] - queue_name: logs - when: - cluster_available: ["$[[CLUSTER_ID]]"] -""", - "version": 1 - } -} - -POST .infini_configs/_doc/ingest_config_tpl -{ - "id": "ingest_config_tpl", - "updated": "2023-10-19T14:49:56.768754+08:00", - "metadata": { - "category": "app_settings", - "name": "agent", - "labels": { - "instance": "_all" - } - }, - "payload": { - "name": "ingest_config.tpl", - "location": "ingest_config.tpl", - "content": """elasticsearch: - - name: $[[INGEST_CLUSTER_ID]] - enabled: true - endpoints: $[[INGEST_CLUSTER_ENDPOINT]] - discovery: - enabled: false - basic_auth: - username: $[[INGEST_CLUSTER_USERNAME]] - password: $[[keystore.ingest_cluster_password]] - -metrics: - enabled: true - queue: metrics - network: - enabled: true - summary: true - details: true - memory: - metrics: - - swap - - memory - disk: - metrics: - - iops - - usage - cpu: - metrics: - - idle - - system - - user - - iowait - - load - instance: - enabled: true - -elastic: - availability_check: - enabled: false - -pipeline: - - name: merge_logs - auto_start: true - keep_running: true - processor: - - indexing_merge: - elasticsearch: "$[[INGEST_CLUSTER_ID]]" - index_name: ".infini_logs" - type_name: "_doc" - input_queue: "logs" - idle_timeout_in_seconds: 10 - output_queue: - name: "merged_requests" - worker_size: 1 - bulk_size_in_mb: 5 - - name: merge_metrics - auto_start: true - keep_running: true - processor: - - indexing_merge: - elasticsearch: "$[[INGEST_CLUSTER_ID]]" - index_name: ".infini_metrics" - type_name: "_doc" - input_queue: "metrics" - output_queue: - name: "merged_requests" - worker_size: 1 - bulk_size_in_mb: 5 - - name: ingest_merged_requests - enabled: true - auto_start: true - keep_running: true - processor: - - bulk_indexing: - max_worker_size: 1 - verbose_bulk_result: false - bulk: - batch_size_in_mb: 5 - batch_size_in_docs: 5000 - max_retry_times: 0 - invalid_queue: "" - response_handle: - include_index_stats: false - include_action_stats: false - output_bulk_stats: false - include_error_details: true - save_error_results: true - save_success_results: false - save_busy_results: false - consumer: - fetch_max_messages: 5 - queues: - type: indexing_merge - when: - cluster_available: ["$[[INGEST_CLUSTER_ID]]"] -""", - "version": 1 - } -} - - -POST .infini_configs/_doc/system_ingest_config_yml -{ - "id": "system_ingest_config_yml", - "updated": "2023-10-19T14:49:56.768754+08:00", - "metadata": { - "category": "app_settings", - "name": "agent", - "labels": { - "instance": "_all" - } - }, - "payload": { - "name": "system_ingest_config.yml", - "location": "system_ingest_config.yml", - "content": """configs.template: - - name: "default_ingest_config" - path: ./config/ingest_config.tpl - variable: - INGEST_CLUSTER_ID: infini_default_ingest_cluster - INGEST_CLUSTER_ENDPOINT: [ "https://localhost:9200" ] - INGEST_CLUSTER_USERNAME: "admin" - CLUSTER_VER: "1.6.0" - CLUSTER_DISTRIBUTION: "easysearch" - INDEX_PREFIX: ".infini_" - -""", - "version": 1 - } -} - +} \ No newline at end of file From 5b9e76280f1b6ca66d2c5c5e46193917806f5cdf Mon Sep 17 00:00:00 2001 From: hardy Date: Sat, 21 Oct 2023 14:07:35 +0800 Subject: [PATCH 18/36] fix case macos tar error when overwrite --- config/install_agent.tpl | 45 +++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/config/install_agent.tpl b/config/install_agent.tpl index 610eaefe..2ca489d9 100644 --- a/config/install_agent.tpl +++ b/config/install_agent.tpl @@ -52,19 +52,31 @@ function print_footprint() { echo "" } -__try() { +function __try() { if [[ $try_status -eq 0 ]]; then ! exception=$( $@ 2>&1 >/dev/null ) try_status=${PIPESTATUS[0]} fi } -__catch() { +function __catch() { _old_try=$try_status try_status=0 [[ $_old_try -ne 0 ]] } +function confirm() { + display_str=$1 + default_ans=$2 + if [[ $default_ans == 'y/N' ]]; then + must_match='[yY]' + else + must_match='[nN]' + fi + read -p"${display_str} [${default_ans}]:" ans + [[ $ans == $must_match ]] +} + function get_latest_version() { echo $(curl -m3 -s "https://release.infinilabs.com/.latest" |sed 's/",/"/;s/"//g;s/://1' |grep -Ev '^[{}]' |grep "$program_name" |awk '{print $NF}') } @@ -82,9 +94,11 @@ function check_dir() { echo -e "Error: The installation directory ${install_dir} should be owner by current user.\nsudo chown -R \$(whoami) ${install_dir}" >&2; exit 1; fi - #if [[ "$(ls -A ${install_dir})" ]]; then - # echo "Error: The installation directory ${install_dir} should be clean." >&2; exit 1; - #fi + if [[ "$(ls -A ${install_dir})" ]]; then + confirm "RISK WARN: Replace or upgrade exists agent version, Proceed?" 'y/N' && echo || exit 1; + uninstall_service + rm -rf ${install_dir}/* + fi } function check_platform() { @@ -283,7 +297,7 @@ node: EOF } -function install_service() { +function uninstall_service() { agent_svc=${install_dir}/${program_name}-${file_ext%%.*} chmod 755 $agent_svc @@ -295,18 +309,28 @@ function install_service() { $agent_svc -service stop &>/dev/null $agent_svc -service uninstall &>/dev/null fi + sleep 3 +} +function install_service() { + agent_svc=${install_dir}/${program_name}-${file_ext%%.*} + chmod 755 $agent_svc echo "[agent] waiting service install & start" $agent_svc -service install &>/dev/null $agent_svc -service start &>/dev/null - sleep 5 + sleep 3 } function register_agent() { - token={{token}} console_endpoint="{{console_endpoint}}" - echo "[agent] waiting registering to INFINI Console" - __try curl -s --retry 1 --retry-delay 3 -m30 -XPOST -o ${install_dir}/setup.log "${console_endpoint}/agent/instance?token=${token}" + token={{token}} + echo '[agent] waiting registering to INFINI Console' + until curl -s -m30 -XPOST "${console_endpoint}/agent/instance?token=${token}"; + do + echo -n '.'; sleep 3; + done; + echo + #__try curl -s --retry 1 --retry-delay 3 -m30 -XPOST -o ${install_dir}/setup.log "${console_endpoint}/agent/instance?token=${token}" } function main() { @@ -337,6 +361,7 @@ function main() { install_binary install_certs install_config + uninstall_service install_service register_agent From bfc82bbf1a48a09a9872ae10ac578bfad851cba1 Mon Sep 17 00:00:00 2001 From: medcl Date: Mon, 23 Oct 2023 12:02:36 +0800 Subject: [PATCH 19/36] ignore malformed json --- config/initialization.tpl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/initialization.tpl b/config/initialization.tpl index 1d795676..3e7c29bb 100644 --- a/config/initialization.tpl +++ b/config/initialization.tpl @@ -83,7 +83,9 @@ PUT _template/$[[INDEX_PREFIX]]metrics-rollover }, "codec" : "best_compression", "number_of_shards" : "1", - "translog.durability":"async" + "translog.durability":"async", + "index.mapping.coerce": false, + "index.mapping.ignore_malformed": true } }, "mappings" : { From 7e4f3c9e4fe9722fd31458aceb44135dc5e31d16 Mon Sep 17 00:00:00 2001 From: medcl Date: Mon, 23 Oct 2023 21:12:30 +0800 Subject: [PATCH 20/36] fix remote config --- modules/agent/api/remote_config.go | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/modules/agent/api/remote_config.go b/modules/agent/api/remote_config.go index 7660c2ae..51a3eb45 100644 --- a/modules/agent/api/remote_config.go +++ b/modules/agent/api/remote_config.go @@ -8,11 +8,14 @@ import ( "bytes" "fmt" "infini.sh/framework/core/elastic" + "infini.sh/framework/core/global" + "infini.sh/framework/core/kv" "infini.sh/framework/core/model" "infini.sh/framework/core/orm" "infini.sh/framework/core/util" common2 "infini.sh/framework/modules/elastic/common" "infini.sh/framework/plugins/managed/common" + log "github.com/cihub/seelog" "time" ) @@ -94,12 +97,28 @@ func dynamicAgentConfigProvider(instance model.Instance) []*common.ConfigFile { } } + + + if len(ids) > 0 { cfg := common.ConfigFile{} cfg.Name = "generated_metrics_tasks.yml" cfg.Location = "generated_metrics_tasks.yml" cfg.Content = getAgentIngestConfigs(ids) + + hash:=util.MD5digest(cfg.Content) + //if local's hash is different from remote's hash, then update local's hash, update version to current timestamp + v,err:=kv.GetValue(LastAgentHash, []byte(global.Env().SystemConfig.NodeConfig.ID)) + if err!=nil||v==nil||string(v)!=hash{ + err:=kv.AddValue(LastAgentHash, []byte(global.Env().SystemConfig.NodeConfig.ID), []byte(hash)) + if err!=nil{ + panic(err) + } + latestTimestamp=time.Now().Unix() + log.Error("local hash is different from remote's hash, update local's hash, update version to current timestamp") + } + cfg.Size = int64(len(cfg.Content)) cfg.Version = latestTimestamp cfg.Managed = true @@ -107,6 +126,8 @@ func dynamicAgentConfigProvider(instance model.Instance) []*common.ConfigFile { result = append(result, &cfg) } + + return result } @@ -161,11 +182,12 @@ func getAgentIngestConfigs(items map[string]BindingItem) string { } + //password: $[[keystore.$[[CLUSTER_ID]]_password]] buffer.WriteString("\n") buffer.WriteString(fmt.Sprintf("#MANAGED_CONFIG_VERSION: %v\n#MANAGED: true\n",latestVersion)) - //password: $[[keystore.$[[CLUSTER_ID]]_password]] - return buffer.String() } +const LastAgentHash ="last_agent_hash" + From e70f7ed90049aa9267e91b67f6549c03068d8921 Mon Sep 17 00:00:00 2001 From: medcl Date: Mon, 23 Oct 2023 21:45:20 +0800 Subject: [PATCH 21/36] sort config files first --- modules/agent/api/remote_config.go | 41 +++++++++++++++++------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/modules/agent/api/remote_config.go b/modules/agent/api/remote_config.go index 51a3eb45..bd23ac63 100644 --- a/modules/agent/api/remote_config.go +++ b/modules/agent/api/remote_config.go @@ -7,6 +7,7 @@ package api import ( "bytes" "fmt" + log "github.com/cihub/seelog" "infini.sh/framework/core/elastic" "infini.sh/framework/core/global" "infini.sh/framework/core/kv" @@ -15,7 +16,6 @@ import ( "infini.sh/framework/core/util" common2 "infini.sh/framework/modules/elastic/common" "infini.sh/framework/plugins/managed/common" - log "github.com/cihub/seelog" "time" ) @@ -97,9 +97,6 @@ func dynamicAgentConfigProvider(instance model.Instance) []*common.ConfigFile { } } - - - if len(ids) > 0 { cfg := common.ConfigFile{} @@ -107,16 +104,16 @@ func dynamicAgentConfigProvider(instance model.Instance) []*common.ConfigFile { cfg.Location = "generated_metrics_tasks.yml" cfg.Content = getAgentIngestConfigs(ids) - hash:=util.MD5digest(cfg.Content) + hash := util.MD5digest(cfg.Content) //if local's hash is different from remote's hash, then update local's hash, update version to current timestamp - v,err:=kv.GetValue(LastAgentHash, []byte(global.Env().SystemConfig.NodeConfig.ID)) - if err!=nil||v==nil||string(v)!=hash{ - err:=kv.AddValue(LastAgentHash, []byte(global.Env().SystemConfig.NodeConfig.ID), []byte(hash)) - if err!=nil{ + v, err := kv.GetValue(LastAgentHash, []byte(global.Env().SystemConfig.NodeConfig.ID)) + if err != nil || v == nil || string(v) != hash { + err := kv.AddValue(LastAgentHash, []byte(global.Env().SystemConfig.NodeConfig.ID), []byte(hash)) + if err != nil { panic(err) } - latestTimestamp=time.Now().Unix() - log.Error("local hash is different from remote's hash, update local's hash, update version to current timestamp") + latestTimestamp = time.Now().Unix() + log.Info("local hash is different from remote's hash, update local's hash, update version to current timestamp") } cfg.Size = int64(len(cfg.Content)) @@ -126,8 +123,6 @@ func dynamicAgentConfigProvider(instance model.Instance) []*common.ConfigFile { result = append(result, &cfg) } - - return result } @@ -135,8 +130,20 @@ func getAgentIngestConfigs(items map[string]BindingItem) string { buffer := bytes.NewBuffer([]byte("configs.template: ")) + //sort items + newItems := []util.KeyValue{} + + for k, v := range items { + newItems = append(newItems, util.KeyValue{Key: k, Value: v.Updated,Payload: v}) + } + + newItems=util.SortKeyValueArray(newItems,false) + var latestVersion int64 - for _, v := range items { + for _, x := range newItems { + + v,ok:=x.Payload.(BindingItem) + if !ok{continue} if v.ClusterID == "" { panic("cluster id is empty") @@ -181,13 +188,11 @@ func getAgentIngestConfigs(items map[string]BindingItem) string { "NODE_LOGS_PATH: \"%v\"\n\n\n", v.NodeUUID, v.ClusterID, clusterEndPoint, username, password, clusterLevelEnabled, nodeLevelEnabled, v.PathLogs))) } - //password: $[[keystore.$[[CLUSTER_ID]]_password]] buffer.WriteString("\n") - buffer.WriteString(fmt.Sprintf("#MANAGED_CONFIG_VERSION: %v\n#MANAGED: true\n",latestVersion)) + buffer.WriteString(fmt.Sprintf("#MANAGED_CONFIG_VERSION: %v\n#MANAGED: true\n", latestVersion)) return buffer.String() } -const LastAgentHash ="last_agent_hash" - +const LastAgentHash = "last_agent_hash" From 0830442bfc7d70811f78cfecbc0a799f9b39a4a1 Mon Sep 17 00:00:00 2001 From: medcl Date: Mon, 23 Oct 2023 22:03:12 +0800 Subject: [PATCH 22/36] disable soft delete by default --- config/install_agent.tpl | 1 + 1 file changed, 1 insertion(+) diff --git a/config/install_agent.tpl b/config/install_agent.tpl index 2ca489d9..aae8c9f4 100644 --- a/config/install_agent.tpl +++ b/config/install_agent.tpl @@ -284,6 +284,7 @@ configs: interval: "10s" servers: # config servers - "http://localhost:9000" + soft_delete: false max_backup_files: 5 tls: #for mTLS connection with config servers enabled: true From 90280ac243642fe78dfe93edae56b68ff85afed7 Mon Sep 17 00:00:00 2001 From: medcl Date: Mon, 23 Oct 2023 22:03:42 +0800 Subject: [PATCH 23/36] limit target traffic --- config/init_agent_config.tpl | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/config/init_agent_config.tpl b/config/init_agent_config.tpl index 70f07ddb..c669c34e 100644 --- a/config/init_agent_config.tpl +++ b/config/init_agent_config.tpl @@ -27,6 +27,11 @@ elasticsearch: basic_auth: username: $[[CLUSTER_USERNAME]] password: $[[CLUSTER_PASSWORD]] + traffic_control: + enabled: true + max_qps_per_node: 100 + max_bytes_per_node: 10485760 + max_connection_per_node: 5 pipeline: #clsuter level metrics @@ -129,6 +134,11 @@ POST .infini_configs/_doc/ingest_config_tpl basic_auth: username: $[[INGEST_CLUSTER_USERNAME]] password: $[[INGEST_CLUSTER_PASSWORD]] + traffic_control: + enabled: true + max_qps_per_node: 1000 + max_bytes_per_node: 10485760 + max_connection_per_node: 10 metrics: enabled: true @@ -240,7 +250,7 @@ POST .infini_configs/_doc/system_ingest_config_yml path: ./config/ingest_config.tpl variable: INGEST_CLUSTER_ID: infini_default_ingest_cluster - INGEST_CLUSTER_ENDPOINT: [ "http://10.0.0.3:7102" ] + INGEST_CLUSTER_ENDPOINT: [ "http://192.168.3.185:9209" ] INGEST_CLUSTER_USERNAME: "admin" INGEST_CLUSTER_PASSWORD: "admin" CLUSTER_VER: "1.6.0" From 425dfd3dfa27fb5e7962fc0ba5192c2f3318a2b6 Mon Sep 17 00:00:00 2001 From: medcl Date: Mon, 23 Oct 2023 22:03:57 +0800 Subject: [PATCH 24/36] update local config samples --- config_repo/settings.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/config_repo/settings.yml b/config_repo/settings.yml index 722d8df2..1a27d978 100644 --- a/config_repo/settings.yml +++ b/config_repo/settings.yml @@ -4,17 +4,17 @@ configs: #define configs group - ./templates/ingest_config.tpl - ./templates/task_config.tpl - ./configs/ingest_config.yml -instances: #define which config instance should fetch - _all: #instance group +#instances: #define which config instance should fetch +# _all: #instance group # plugins: # - ingest # instances: # - ck0mkk805f5virpsejp0 # - ckjrpdg05f5lrfp8qlng - configs: - - general_ingest_template - secrets: - - ingest_cluster_password +# configs: +# - general_ingest_template +# secrets: +# - ingest_cluster_password secrets: ingest_cluster_password: #group name From 9fa8658eb94d1a121c308debdbb5623bb0efba5e Mon Sep 17 00:00:00 2001 From: medcl Date: Tue, 24 Oct 2023 11:19:38 +0800 Subject: [PATCH 25/36] update agent template, collect more network metrics --- config/init_agent_config.tpl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/config/init_agent_config.tpl b/config/init_agent_config.tpl index c669c34e..24c9aa97 100644 --- a/config/init_agent_config.tpl +++ b/config/init_agent_config.tpl @@ -146,6 +146,8 @@ metrics: network: enabled: true summary: true + sockets: true + throughput: true details: true memory: metrics: @@ -250,7 +252,7 @@ POST .infini_configs/_doc/system_ingest_config_yml path: ./config/ingest_config.tpl variable: INGEST_CLUSTER_ID: infini_default_ingest_cluster - INGEST_CLUSTER_ENDPOINT: [ "http://192.168.3.185:9209" ] + INGEST_CLUSTER_ENDPOINT: [ "http://10.0.0.3:7102" ] INGEST_CLUSTER_USERNAME: "admin" INGEST_CLUSTER_PASSWORD: "admin" CLUSTER_VER: "1.6.0" @@ -258,7 +260,7 @@ POST .infini_configs/_doc/system_ingest_config_yml INDEX_PREFIX: ".infini_" """, - "version": 2 + "version": 3 } } From 6672f545b7542022f35cace1646597bee2459902 Mon Sep 17 00:00:00 2001 From: medcl Date: Tue, 24 Oct 2023 19:47:31 +0800 Subject: [PATCH 26/36] improve nodes discovery --- modules/agent/api/elasticsearch.go | 58 ++++++++++++++++++++++++++++++ modules/agent/api/remote_config.go | 33 +++++++++++------ 2 files changed, 80 insertions(+), 11 deletions(-) diff --git a/modules/agent/api/elasticsearch.go b/modules/agent/api/elasticsearch.go index 13bf827d..65c74aac 100644 --- a/modules/agent/api/elasticsearch.go +++ b/modules/agent/api/elasticsearch.go @@ -14,7 +14,9 @@ import ( "infini.sh/framework/core/model" "infini.sh/framework/core/orm" "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic/adapter" "infini.sh/framework/modules/elastic/common" + "infini.sh/framework/modules/elastic/metadata" "infini.sh/framework/plugins/managed/server" "net/http" "time" @@ -90,14 +92,70 @@ func refreshNodesInfo(inst *model.Instance) (*elastic.DiscoveryResult, error) { return nil, fmt.Errorf("error on get nodes info from agent: %w", err) } + newNodes := map[string]*elastic.LocalNodeInfo{} + //binding nodes info with agent for nodeID, node := range nodesInfo.Nodes { v, ok := enrolledNodesByAgent[nodeID] + node.Status = "online" if ok { node.ClusterID = v.ClusterID node.Enrolled = true + + //output + newNodes[nodeID] = node + } else { + newNodes[nodeID] = node } } + //TODO, merge requests to one + for k, v := range enrolledNodesByAgent { + if _, ok := newNodes[k]; !ok { + client := elastic.GetClient(v.ClusterID) + status := "online" + nodeInfo, err := client.GetNodeInfo(v.NodeUUID) + var clusterInfo *elastic.ClusterInformation + if err != nil ||nodeInfo == nil { + status= "offline" + + //get nodes information + nodeInfos, err := metadata.GetNodeInformation(v.ClusterID,[]string{v.NodeUUID}) + if err!=nil||len(nodeInfos)==0{ + log.Error("node info not found:",v.ClusterID,",",[]string{v.NodeUUID},",",err,err!=nil,len(nodeInfos)==0) + continue + } + + nodeInfo=nodeInfos[0] + + //get cluster information + clusterInfo,err=metadata.GetClusterInformation(v.ClusterID) + if err!=nil||clusterInfo==nil{ + log.Error("cluster info not found:",v.ClusterID,",",err,clusterInfo==nil) + continue + } + + + }else{ + clusterInfo, err = adapter.ClusterVersion(elastic.GetMetadata(v.ClusterID)) + if err != nil || clusterInfo == nil{ + log.Error(err) + continue + } + } + + newNodes[k] = &elastic.LocalNodeInfo{ + Status: status, + ClusterID: v.ClusterID, + NodeUUID: v.NodeUUID, + Enrolled: true, + NodeInfo: nodeInfo, + ClusterInfo: clusterInfo, + } + } + } + + nodesInfo.Nodes = newNodes + return nodesInfo, nil } diff --git a/modules/agent/api/remote_config.go b/modules/agent/api/remote_config.go index bd23ac63..5e3467a0 100644 --- a/modules/agent/api/remote_config.go +++ b/modules/agent/api/remote_config.go @@ -102,18 +102,21 @@ func dynamicAgentConfigProvider(instance model.Instance) []*common.ConfigFile { cfg := common.ConfigFile{} cfg.Name = "generated_metrics_tasks.yml" cfg.Location = "generated_metrics_tasks.yml" - cfg.Content = getAgentIngestConfigs(ids) + cfg.Content, cfg.Hash = getAgentIngestConfigs(instance.ID, ids) - hash := util.MD5digest(cfg.Content) + hash := cfg.Hash + if cfg.Hash == "" { + hash = util.MD5digest(cfg.Content) + } //if local's hash is different from remote's hash, then update local's hash, update version to current timestamp - v, err := kv.GetValue(LastAgentHash, []byte(global.Env().SystemConfig.NodeConfig.ID)) + v, err := kv.GetValue(LastAgentHash, []byte(global.Env().SystemConfig.NodeConfig.ID+":"+instance.ID)) if err != nil || v == nil || string(v) != hash { - err := kv.AddValue(LastAgentHash, []byte(global.Env().SystemConfig.NodeConfig.ID), []byte(hash)) + err := kv.AddValue(LastAgentHash, []byte(global.Env().SystemConfig.NodeConfig.ID+":"+instance.ID), []byte(hash)) if err != nil { panic(err) } latestTimestamp = time.Now().Unix() - log.Info("local hash is different from remote's hash, update local's hash, update version to current timestamp") + log.Infof("hash: %v vs %v, update version to current timestamp: %v", string(v), hash, latestTimestamp) } cfg.Size = int64(len(cfg.Content)) @@ -126,7 +129,11 @@ func dynamicAgentConfigProvider(instance model.Instance) []*common.ConfigFile { return result } -func getAgentIngestConfigs(items map[string]BindingItem) string { +func getAgentIngestConfigs(instance string, items map[string]BindingItem) (string, string) { + + if instance == "" { + panic("instance id is empty") + } buffer := bytes.NewBuffer([]byte("configs.template: ")) @@ -134,16 +141,18 @@ func getAgentIngestConfigs(items map[string]BindingItem) string { newItems := []util.KeyValue{} for k, v := range items { - newItems = append(newItems, util.KeyValue{Key: k, Value: v.Updated,Payload: v}) + newItems = append(newItems, util.KeyValue{Key: k, Value: v.Updated, Payload: v}) } - newItems=util.SortKeyValueArray(newItems,false) + newItems = util.SortKeyValueArray(newItems, false) var latestVersion int64 for _, x := range newItems { - v,ok:=x.Payload.(BindingItem) - if !ok{continue} + v, ok := x.Payload.(BindingItem) + if !ok { + continue + } if v.ClusterID == "" { panic("cluster id is empty") @@ -188,11 +197,13 @@ func getAgentIngestConfigs(items map[string]BindingItem) string { "NODE_LOGS_PATH: \"%v\"\n\n\n", v.NodeUUID, v.ClusterID, clusterEndPoint, username, password, clusterLevelEnabled, nodeLevelEnabled, v.PathLogs))) } + hash := util.MD5digest(buffer.String()) + //password: $[[keystore.$[[CLUSTER_ID]]_password]] buffer.WriteString("\n") buffer.WriteString(fmt.Sprintf("#MANAGED_CONFIG_VERSION: %v\n#MANAGED: true\n", latestVersion)) - return buffer.String() + return buffer.String(), hash } const LastAgentHash = "last_agent_hash" From 7fa78f47ec4abdc715106f0841bae091a460d42d Mon Sep 17 00:00:00 2001 From: medcl Date: Tue, 24 Oct 2023 23:28:24 +0800 Subject: [PATCH 27/36] fix discovery --- modules/agent/api/elasticsearch.go | 29 ++++++++++++++++++++++++----- modules/agent/api/init.go | 1 + 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/modules/agent/api/elasticsearch.go b/modules/agent/api/elasticsearch.go index 65c74aac..77463934 100644 --- a/modules/agent/api/elasticsearch.go +++ b/modules/agent/api/elasticsearch.go @@ -108,10 +108,17 @@ func refreshNodesInfo(inst *model.Instance) (*elastic.DiscoveryResult, error) { } } + var findPIDS=map[int]*elastic.NodesInfo{} + //TODO, merge requests to one for k, v := range enrolledNodesByAgent { + if _, ok := newNodes[k]; !ok { - client := elastic.GetClient(v.ClusterID) + client := elastic.GetClientNoPanic(v.ClusterID) + if client == nil { + log.Error("client not found:", v.ClusterID) + continue + } status := "online" nodeInfo, err := client.GetNodeInfo(v.NodeUUID) var clusterInfo *elastic.ClusterInformation @@ -133,8 +140,6 @@ func refreshNodesInfo(inst *model.Instance) (*elastic.DiscoveryResult, error) { log.Error("cluster info not found:",v.ClusterID,",",err,clusterInfo==nil) continue } - - }else{ clusterInfo, err = adapter.ClusterVersion(elastic.GetMetadata(v.ClusterID)) if err != nil || clusterInfo == nil{ @@ -143,6 +148,8 @@ func refreshNodesInfo(inst *model.Instance) (*elastic.DiscoveryResult, error) { } } + findPIDS[nodeInfo.Process.Id]=nodeInfo + newNodes[k] = &elastic.LocalNodeInfo{ Status: status, ClusterID: v.ClusterID, @@ -155,7 +162,14 @@ func refreshNodesInfo(inst *model.Instance) (*elastic.DiscoveryResult, error) { } nodesInfo.Nodes = newNodes + newUnknows:=[]model.ProcessInfo{} + for _,v:=range nodesInfo.UnknownProcess{ + if _,ok:=findPIDS[v.PID];!ok{ + newUnknows=append(newUnknows,v) + } + } + nodesInfo.UnknownProcess=newUnknows return nodesInfo, nil } @@ -362,6 +376,10 @@ type ClusterInfo struct { ClusterIDs []string `json:"cluster_id"` } +func (h *APIHandler) autoEnrollESNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + //TODO +} + func (h *APIHandler) discoveryESNodesInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { id := ps.MustGetParameter("instance_id") @@ -385,12 +403,12 @@ func (h *APIHandler) discoveryESNodesInfo(w http.ResponseWriter, req *http.Reque if len(nodes.UnknownProcess) > 0 { var discoveredPIDs map[int]*elastic.LocalNodeInfo = make(map[int]*elastic.LocalNodeInfo) - if req.Method == "POST" { bytes, err := h.GetRawBody(req) if err != nil { panic(err) } + if len(bytes) > 0 { clusterInfo := ClusterInfo{} util.FromJSONBytes(bytes, &clusterInfo) @@ -418,8 +436,9 @@ func (h *APIHandler) discoveryESNodesInfo(w http.ResponseWriter, req *http.Reque //try https again success, tryAgain, nodeInfo = h.getESNodeInfoViaProxy(nodeHost, "https", auth, &instance) } + if success { - log.Error("connect to es node success:", nodeHost) + log.Debug("connect to es node success:", nodeHost,", pid: ",node.PID) discoveredPIDs[node.PID] = nodeInfo break } diff --git a/modules/agent/api/init.go b/modules/agent/api/init.go index b1a25c7d..5cb4328a 100644 --- a/modules/agent/api/init.go +++ b/modules/agent/api/init.go @@ -24,6 +24,7 @@ func Init() { //bind agent with nodes api.HandleAPIMethod(api.GET, "/instance/:instance_id/node/_discovery", handler.RequirePermission(handler.discoveryESNodesInfo, enum.PermissionAgentInstanceRead)) api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_discovery", handler.RequirePermission(handler.discoveryESNodesInfo, enum.PermissionAgentInstanceRead)) + api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_auto_enroll", handler.RequirePermission(handler.autoEnrollESNode, enum.PermissionAgentInstanceWrite)) api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_enroll", handler.RequirePermission(handler.enrollESNode, enum.PermissionAgentInstanceWrite)) api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_revoke", handler.RequirePermission(handler.revokeESNode, enum.PermissionAgentInstanceWrite)) From 833748b9e807fd651a2c6773675b069656c23661 Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 25 Oct 2023 09:02:46 +0800 Subject: [PATCH 28/36] ignore invalid clusters --- modules/agent/api/remote_config.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/agent/api/remote_config.go b/modules/agent/api/remote_config.go index 5e3467a0..6ebc8049 100644 --- a/modules/agent/api/remote_config.go +++ b/modules/agent/api/remote_config.go @@ -159,6 +159,10 @@ func getAgentIngestConfigs(instance string, items map[string]BindingItem) (strin } metadata := elastic.GetMetadata(v.ClusterID) + if metadata == nil || metadata.Config == nil{ + log.Errorf("metadata is nil: %v",v.ClusterID) + continue + } var clusterLevelEnabled = false var nodeLevelEnabled = true var clusterEndPoint = metadata.Config.GetAnyEndpoint() From 36cbc265f8e6f1b915a0b0eb7a6b1e0b6dbe1216 Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 25 Oct 2023 11:37:32 +0800 Subject: [PATCH 29/36] remove cluster level tasks from agent task template --- config/init_agent_config.tpl | 59 ++++++------------------------------ 1 file changed, 9 insertions(+), 50 deletions(-) diff --git a/config/init_agent_config.tpl b/config/init_agent_config.tpl index 24c9aa97..a2c57e51 100644 --- a/config/init_agent_config.tpl +++ b/config/init_agent_config.tpl @@ -18,8 +18,8 @@ POST .infini_configs/_doc/task_config_tpl CLUSTER_PASSWORD: $[[keystore.$[[CLUSTER_ID]]_password]] elasticsearch: - - id: $[[CLUSTER_ID]] - name: $[[CLUSTER_ID]] + - id: $[[TASK_ID]] + name: $[[TASK_ID]] enabled: true endpoints: $[[CLUSTER_ENDPOINT]] discovery: @@ -34,83 +34,42 @@ elasticsearch: max_connection_per_node: 5 pipeline: -#clsuter level metrics -- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] - enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] - keep_running: true - singleton: true - name: collect_$[[CLUSTER_ID]]_es_cluster_stats - retry_delay_in_ms: 10000 - processor: - - es_cluster_stats: - elasticsearch: $[[CLUSTER_ID]] - labels: - cluster_id: $[[CLUSTER_ID]] - when: - cluster_available: ["$[[CLUSTER_ID]]"] - -- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] - enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] - keep_running: true - singleton: true - name: collect_$[[CLUSTER_ID]]_es_index_stats - retry_delay_in_ms: 10000 - processor: - - es_index_stats: - elasticsearch: $[[CLUSTER_ID]] - labels: - cluster_id: $[[CLUSTER_ID]] - when: - cluster_available: ["$[[CLUSTER_ID]]"] - -- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] - enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] - keep_running: true - singleton: true - name: collect_$[[CLUSTER_ID]]_es_cluster_health - retry_delay_in_ms: 10000 - processor: - - es_cluster_health: - elasticsearch: $[[CLUSTER_ID]] - labels: - cluster_id: $[[CLUSTER_ID]] - when: - cluster_available: ["$[[CLUSTER_ID]]"] #node level metrics - auto_start: $[[NODE_LEVEL_TASKS_ENABLED]] enabled: $[[NODE_LEVEL_TASKS_ENABLED]] keep_running: true - name: collect_$[[CLUSTER_ID]]_es_node_stats + name: collect_$[[TASK_ID]]_es_node_stats retry_delay_in_ms: 10000 processor: - es_node_stats: - elasticsearch: $[[CLUSTER_ID]] + elasticsearch: $[[TASK_ID]] labels: cluster_id: $[[CLUSTER_ID]] when: - cluster_available: ["$[[CLUSTER_ID]]"] + cluster_available: ["$[[TASK_ID]]"] #node logs - auto_start: $[[NODE_LEVEL_TASKS_ENABLED]] enabled: $[[NODE_LEVEL_TASKS_ENABLED]] keep_running: true - name: collect_$[[CLUSTER_ID]]_es_logs + name: collect_$[[TASK_ID]]_es_logs retry_delay_in_ms: 10000 processor: - es_logs_processor: - elasticsearch: $[[CLUSTER_ID]] + elasticsearch: $[[TASK_ID]] labels: cluster_id: $[[CLUSTER_ID]] logs_path: $[[NODE_LOGS_PATH]] queue_name: logs when: - cluster_available: ["$[[CLUSTER_ID]]"] + cluster_available: ["$[[TASK_ID]]"] """, "version": 1 } } +#system ingest template POST .infini_configs/_doc/ingest_config_tpl { "id": "ingest_config_tpl", From 4415e63aa4ae84f86f92ce51eef3bab6540afdf8 Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 25 Oct 2023 11:38:10 +0800 Subject: [PATCH 30/36] fix multi node tasks --- modules/agent/api/remote_config.go | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/modules/agent/api/remote_config.go b/modules/agent/api/remote_config.go index 6ebc8049..674552c6 100644 --- a/modules/agent/api/remote_config.go +++ b/modules/agent/api/remote_config.go @@ -155,7 +155,8 @@ func getAgentIngestConfigs(instance string, items map[string]BindingItem) (strin } if v.ClusterID == "" { - panic("cluster id is empty") + log.Error("cluster id is empty") + continue } metadata := elastic.GetMetadata(v.ClusterID) @@ -165,7 +166,6 @@ func getAgentIngestConfigs(instance string, items map[string]BindingItem) (strin } var clusterLevelEnabled = false var nodeLevelEnabled = true - var clusterEndPoint = metadata.Config.GetAnyEndpoint() var username = "" var password = "" @@ -173,12 +173,14 @@ func getAgentIngestConfigs(instance string, items map[string]BindingItem) (strin if metadata.Config.AgentCredentialID != "" { credential, err := common2.GetCredential(metadata.Config.AgentCredentialID) if err != nil { - panic(err) + log.Error(err) + continue } var dv interface{} dv, err = credential.Decode() if err != nil { - panic(err) + log.Error(err) + continue } if auth, ok := dv.(model.BasicAuth); ok { username = auth.Username @@ -186,19 +188,31 @@ func getAgentIngestConfigs(instance string, items map[string]BindingItem) (strin } } + if v.PublishAddress==""{ + log.Errorf("publish address is empty: %v",v.NodeUUID) + continue + } + + nodeEndPoint := metadata.PrepareEndpoint(v.PublishAddress) + if v.Updated > latestVersion { latestVersion = v.Updated } + taskID:=v.ClusterID+"_"+v.NodeUUID + buffer.Write([]byte(fmt.Sprintf("\n - name: \"%v\"\n path: ./config/task_config.tpl\n "+ "variable:\n "+ + "TASK_ID: %v\n "+ "CLUSTER_ID: %v\n "+ + "NODE_UUID: %v\n "+ "CLUSTER_ENDPOINT: [\"%v\"]\n "+ "CLUSTER_USERNAME: \"%v\"\n "+ "CLUSTER_PASSWORD: \"%v\"\n "+ "CLUSTER_LEVEL_TASKS_ENABLED: %v\n "+ "NODE_LEVEL_TASKS_ENABLED: %v\n "+ - "NODE_LOGS_PATH: \"%v\"\n\n\n", v.NodeUUID, v.ClusterID, clusterEndPoint, username, password, clusterLevelEnabled, nodeLevelEnabled, v.PathLogs))) + "NODE_LOGS_PATH: \"%v\"\n\n\n", taskID, taskID, + v.ClusterID,v.NodeUUID, nodeEndPoint, username, password, clusterLevelEnabled, nodeLevelEnabled, v.PathLogs))) } hash := util.MD5digest(buffer.String()) From 505a76072b9a09b5021792a27579e63e346b4cbc Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 25 Oct 2023 11:57:53 +0800 Subject: [PATCH 31/36] add cluster_uuid to node metrics --- config/init_agent_config.tpl | 1 + modules/agent/api/remote_config.go | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/init_agent_config.tpl b/config/init_agent_config.tpl index a2c57e51..47ff68d6 100644 --- a/config/init_agent_config.tpl +++ b/config/init_agent_config.tpl @@ -20,6 +20,7 @@ POST .infini_configs/_doc/task_config_tpl elasticsearch: - id: $[[TASK_ID]] name: $[[TASK_ID]] + cluster_uuid: $[[CLUSTER_UUID]] enabled: true endpoints: $[[CLUSTER_ENDPOINT]] discovery: diff --git a/modules/agent/api/remote_config.go b/modules/agent/api/remote_config.go index 674552c6..68c1919c 100644 --- a/modules/agent/api/remote_config.go +++ b/modules/agent/api/remote_config.go @@ -205,6 +205,7 @@ func getAgentIngestConfigs(instance string, items map[string]BindingItem) (strin "variable:\n "+ "TASK_ID: %v\n "+ "CLUSTER_ID: %v\n "+ + "CLUSTER_UUID: %v\n "+ "NODE_UUID: %v\n "+ "CLUSTER_ENDPOINT: [\"%v\"]\n "+ "CLUSTER_USERNAME: \"%v\"\n "+ @@ -212,7 +213,7 @@ func getAgentIngestConfigs(instance string, items map[string]BindingItem) (strin "CLUSTER_LEVEL_TASKS_ENABLED: %v\n "+ "NODE_LEVEL_TASKS_ENABLED: %v\n "+ "NODE_LOGS_PATH: \"%v\"\n\n\n", taskID, taskID, - v.ClusterID,v.NodeUUID, nodeEndPoint, username, password, clusterLevelEnabled, nodeLevelEnabled, v.PathLogs))) + v.ClusterID,v.ClusterUUID,v.NodeUUID, nodeEndPoint, username, password, clusterLevelEnabled, nodeLevelEnabled, v.PathLogs))) } hash := util.MD5digest(buffer.String()) From 2eb8304e68c973dea17ee7cfa3ee6e487b39753b Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 25 Oct 2023 17:02:02 +0800 Subject: [PATCH 32/36] update configs --- config/agent/agent_config.tpl | 34 +++++++ config/{ => agent}/init_agent_config.tpl | 41 ++------ config/agent/init_gateway_config.tpl | 114 +++++++++++++++++++++++ 3 files changed, 154 insertions(+), 35 deletions(-) create mode 100644 config/agent/agent_config.tpl rename config/{ => agent}/init_agent_config.tpl (82%) create mode 100644 config/agent/init_gateway_config.tpl diff --git a/config/agent/agent_config.tpl b/config/agent/agent_config.tpl new file mode 100644 index 00000000..16770879 --- /dev/null +++ b/config/agent/agent_config.tpl @@ -0,0 +1,34 @@ + +#TODO, need to replace cleartext password to keystore, and ingest endpoint +POST .infini_configs/_doc/system_ingest_config_yml +{ + "id": "system_ingest_config_yml", + "updated": "2023-10-18T14:49:56.768754+08:00", + "metadata": { + "category": "app_settings", + "name": "agent", + "labels": { + "instance": "_all" + } + }, + "payload": { + "name": "system_ingest_config.yml", + "location": "system_ingest_config.yml", + "content": """ + +configs.template: + - name: "default_ingest_config" + path: ./config/ingest_config.tpl + variable: + INGEST_CLUSTER_ID: infini_default_ingest_cluster + INGEST_CLUSTER_ENDPOINT: [ "http://192.168.3.185:8000" ] + INGEST_CLUSTER_USERNAME: "ingest" + INGEST_CLUSTER_PASSWORD: "password" + CLUSTER_VER: "1.6.0" + CLUSTER_DISTRIBUTION: "easysearch" + INDEX_PREFIX: ".infini_" + +""", + "version": 3 + } +} diff --git a/config/init_agent_config.tpl b/config/agent/init_agent_config.tpl similarity index 82% rename from config/init_agent_config.tpl rename to config/agent/init_agent_config.tpl index 47ff68d6..2b540a05 100644 --- a/config/init_agent_config.tpl +++ b/config/agent/init_agent_config.tpl @@ -14,7 +14,9 @@ POST .infini_configs/_doc/task_config_tpl "payload": { "name": "task_config.tpl", "location": "task_config.tpl", - "content": """env: + "content": """ + +env: CLUSTER_PASSWORD: $[[keystore.$[[CLUSTER_ID]]_password]] elasticsearch: @@ -85,7 +87,9 @@ POST .infini_configs/_doc/ingest_config_tpl "payload": { "name": "ingest_config.tpl", "location": "ingest_config.tpl", - "content": """elasticsearch: + "content": """ + +elasticsearch: - name: $[[INGEST_CLUSTER_ID]] enabled: true endpoints: $[[INGEST_CLUSTER_ENDPOINT]] @@ -191,36 +195,3 @@ pipeline: } } - -#TODO, need to replace cleartext password to keystore, and ingest endpoint -POST .infini_configs/_doc/system_ingest_config_yml -{ - "id": "system_ingest_config_yml", - "updated": "2023-10-18T14:49:56.768754+08:00", - "metadata": { - "category": "app_settings", - "name": "agent", - "labels": { - "instance": "_all" - } - }, - "payload": { - "name": "system_ingest_config.yml", - "location": "system_ingest_config.yml", - "content": """configs.template: - - name: "default_ingest_config" - path: ./config/ingest_config.tpl - variable: - INGEST_CLUSTER_ID: infini_default_ingest_cluster - INGEST_CLUSTER_ENDPOINT: [ "http://10.0.0.3:7102" ] - INGEST_CLUSTER_USERNAME: "admin" - INGEST_CLUSTER_PASSWORD: "admin" - CLUSTER_VER: "1.6.0" - CLUSTER_DISTRIBUTION: "easysearch" - INDEX_PREFIX: ".infini_" - -""", - "version": 3 - } -} - diff --git a/config/agent/init_gateway_config.tpl b/config/agent/init_gateway_config.tpl new file mode 100644 index 00000000..915ed2fb --- /dev/null +++ b/config/agent/init_gateway_config.tpl @@ -0,0 +1,114 @@ +POST .infini_configs/_doc/agent_relay_gateway_config_yml +{ + "id": "agent_relay_gateway_config_yml", + "updated": "2023-10-19T14:49:56.768754+08:00", + "metadata": { + "category": "app_settings", + "name": "gateway", + "labels": { + "instance": "_all" + } + }, + "payload": { + "name": "agent_relay_gateway_config.yml", + "location": "agent_relay_gateway_config.yml", + "content": """ +env: + CLUSTER_ENDPOINTS: ["http://10.0.0.3:7102"] + CLUSTER_USERNAME: admin + CLUSTER_PASSWORD: admin + +path.data: data +path.logs: log + +entry: + - name: my_es_entry + enabled: true + router: my_router + max_concurrency: 200000 + network: + binding: 0.0.0.0:8000 + +flow: + - name: async_bulk + filter: + - basic_auth: + valid_users: + ingest: password + - bulk_reshuffle: + when: + contains: + _ctx.request.path: /_bulk + elasticsearch: prod + level: cluster + partition_size: 3 + fix_null_id: true + - elasticsearch: + elasticsearch: prod #elasticsearch configure reference name + max_connection_per_node: 1000 #max tcp connection to upstream, default for all nodes + max_response_size: -1 #default for all nodes + balancer: weight + refresh: # refresh upstream nodes list, need to enable this feature to use elasticsearch nodes auto discovery + enabled: true + interval: 60s + filter: + roles: + exclude: + - master + +router: + - name: my_router + default_flow: async_bulk + +elasticsearch: + - name: prod + enabled: true + endpoints: $[[env.CLUSTER_ENDPOINTS]] + discovery: + enabled: false + basic_auth: + username: $[[env.CLUSTER_USERNAME]] + password: $[[env.CLUSTER_PASSWORD]] + traffic_control: + enabled: true + max_qps_per_node: 100 + max_bytes_per_node: 10485760 + max_connection_per_node: 5 + +elastic: + enabled: true + remote_configs: false + elasticsearch: prod + metadata_refresh: + enabled: true + interval: 30s + discovery: + enabled: true + refresh: + enabled: true + interval: 30s + +pipeline: + - name: bulk_request_ingest + auto_start: true + keep_running: true + retry_delay_in_ms: 1000 + processor: + - bulk_indexing: + max_connection_per_node: 100 + num_of_slices: 3 + max_worker_size: 30 + idle_timeout_in_seconds: 10 + bulk: + compress: false + batch_size_in_mb: 10 + batch_size_in_docs: 10000 + consumer: + fetch_max_messages: 100 + queue_selector: + labels: + type: bulk_reshuffle +""", + "version": 1 + } +} \ No newline at end of file From f7fc4c00a3780bc9c002fea29e59fe54aaf859dc Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 25 Oct 2023 17:02:23 +0800 Subject: [PATCH 33/36] filter for only agent --- modules/agent/api/remote_config.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/agent/api/remote_config.go b/modules/agent/api/remote_config.go index 68c1919c..e8f903f6 100644 --- a/modules/agent/api/remote_config.go +++ b/modules/agent/api/remote_config.go @@ -81,6 +81,10 @@ func remoteConfigProvider(instance model.Instance) []*common.ConfigFile { func dynamicAgentConfigProvider(instance model.Instance) []*common.ConfigFile { + if instance.Application.Name != "agent" { + return nil + } + //get config files from remote db //get settings with this agent id From 0cae6007a193d6ee96ccc5bab22c3cab783e7ba8 Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 25 Oct 2023 22:23:45 +0800 Subject: [PATCH 34/36] optimize agent tpl files --- config/agent/init_agent_config.tpl | 5 ++++- config/install_agent.tpl | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/config/agent/init_agent_config.tpl b/config/agent/init_agent_config.tpl index 2b540a05..4693292a 100644 --- a/config/agent/init_agent_config.tpl +++ b/config/agent/init_agent_config.tpl @@ -111,7 +111,7 @@ metrics: enabled: true summary: true sockets: true - throughput: true + #throughput: true details: true memory: metrics: @@ -139,6 +139,7 @@ pipeline: - name: merge_logs auto_start: true keep_running: true + retry_delay_in_ms: 10000 processor: - indexing_merge: elasticsearch: "$[[INGEST_CLUSTER_ID]]" @@ -153,6 +154,7 @@ pipeline: - name: merge_metrics auto_start: true keep_running: true + retry_delay_in_ms: 10000 processor: - indexing_merge: elasticsearch: "$[[INGEST_CLUSTER_ID]]" @@ -167,6 +169,7 @@ pipeline: enabled: true auto_start: true keep_running: true + retry_delay_in_ms: 10000 processor: - bulk_indexing: max_worker_size: 1 diff --git a/config/install_agent.tpl b/config/install_agent.tpl index aae8c9f4..d4a268a7 100644 --- a/config/install_agent.tpl +++ b/config/install_agent.tpl @@ -252,7 +252,7 @@ elastic: disk_queue: max_msg_size: 20485760 max_bytes_per_file: 20485760 - max_used_bytes: 524288000 + max_used_bytes: 1024288000 retention.max_num_of_local_files: 1 compress: idle_threshold: 0 From 1403499c2786aaab12038d9dc56dd3b531a264b0 Mon Sep 17 00:00:00 2001 From: medcl Date: Fri, 27 Oct 2023 16:33:25 +0800 Subject: [PATCH 35/36] allow to directly enroll --- modules/agent/api/elasticsearch.go | 157 +++++++++++++++++++---------- modules/agent/api/init.go | 2 - modules/agent/api/remote_config.go | 17 +++- 3 files changed, 119 insertions(+), 57 deletions(-) diff --git a/modules/agent/api/elasticsearch.go b/modules/agent/api/elasticsearch.go index 77463934..0def2e83 100644 --- a/modules/agent/api/elasticsearch.go +++ b/modules/agent/api/elasticsearch.go @@ -52,12 +52,12 @@ func GetEnrolledNodesByAgent(instance *model.Instance) (map[string]BindingItem, if ok { item := BindingItem{} item.ClusterID = util.ToString(f["cluster_id"]) - item.ClusterName = util.ToString(f["cluster_name"]) + //item.ClusterName = util.ToString(f["cluster_name"]) item.ClusterUUID = util.ToString(f["cluster_uuid"]) - item.PublishAddress = util.ToString(f["publish_address"]) - item.NodeName = util.ToString(f["node_name"]) - item.PathHome = util.ToString(f["path_home"]) - item.PathLogs = util.ToString(f["path_logs"]) + //item.PublishAddress = util.ToString(f["publish_address"]) + //item.NodeName = util.ToString(f["node_name"]) + //item.PathHome = util.ToString(f["path_home"]) + //item.PathLogs = util.ToString(f["path_logs"]) item.NodeUUID = nodeID t, ok := v["updated"] @@ -108,7 +108,7 @@ func refreshNodesInfo(inst *model.Instance) (*elastic.DiscoveryResult, error) { } } - var findPIDS=map[int]*elastic.NodesInfo{} + var findPIDS = map[int]*elastic.NodesInfo{} //TODO, merge requests to one for k, v := range enrolledNodesByAgent { @@ -122,33 +122,38 @@ func refreshNodesInfo(inst *model.Instance) (*elastic.DiscoveryResult, error) { status := "online" nodeInfo, err := client.GetNodeInfo(v.NodeUUID) var clusterInfo *elastic.ClusterInformation - if err != nil ||nodeInfo == nil { - status= "offline" + if err != nil || nodeInfo == nil { + status = "offline" //get nodes information - nodeInfos, err := metadata.GetNodeInformation(v.ClusterID,[]string{v.NodeUUID}) - if err!=nil||len(nodeInfos)==0{ - log.Error("node info not found:",v.ClusterID,",",[]string{v.NodeUUID},",",err,err!=nil,len(nodeInfos)==0) + nodeInfos, err := metadata.GetNodeInformation(v.ClusterID, []string{v.NodeUUID}) + if err != nil || len(nodeInfos) == 0 { + log.Error("node info not found:", v.ClusterID, ",", []string{v.NodeUUID}, ",", err, err != nil, len(nodeInfos) == 0) continue } - nodeInfo=nodeInfos[0] + //get node information + nodeInfo, ok = nodeInfos[v.NodeUUID] + if !ok { + log.Error("node info not found:", v.ClusterID, ",", v.NodeUUID, ",", err) + continue + } //get cluster information - clusterInfo,err=metadata.GetClusterInformation(v.ClusterID) - if err!=nil||clusterInfo==nil{ - log.Error("cluster info not found:",v.ClusterID,",",err,clusterInfo==nil) + clusterInfo, err = metadata.GetClusterInformation(v.ClusterID) + if err != nil || clusterInfo == nil { + log.Error("cluster info not found:", v.ClusterID, ",", err, clusterInfo == nil) continue } - }else{ + } else { clusterInfo, err = adapter.ClusterVersion(elastic.GetMetadata(v.ClusterID)) - if err != nil || clusterInfo == nil{ + if err != nil || clusterInfo == nil { log.Error(err) continue } } - findPIDS[nodeInfo.Process.Id]=nodeInfo + findPIDS[nodeInfo.Process.Id] = nodeInfo newNodes[k] = &elastic.LocalNodeInfo{ Status: status, @@ -162,14 +167,14 @@ func refreshNodesInfo(inst *model.Instance) (*elastic.DiscoveryResult, error) { } nodesInfo.Nodes = newNodes - newUnknows:=[]model.ProcessInfo{} - for _,v:=range nodesInfo.UnknownProcess{ + newUnknows := []model.ProcessInfo{} + for _, v := range nodesInfo.UnknownProcess { - if _,ok:=findPIDS[v.PID];!ok{ - newUnknows=append(newUnknows,v) + if _, ok := findPIDS[v.PID]; !ok { + newUnknows = append(newUnknows, v) } } - nodesInfo.UnknownProcess=newUnknows + nodesInfo.UnknownProcess = newUnknows return nodesInfo, nil } @@ -191,17 +196,18 @@ func GetElasticsearchNodesViaAgent(ctx context.Context, instance *model.Instance } type BindingItem struct { - ClusterName string `json:"cluster_name"` - ClusterUUID string `json:"cluster_uuid"` - NodeUUID string `json:"node_uuid"` - PublishAddress string `json:"publish_address"` - NodeName string `json:"node_name"` - PathLogs string `json:"path_logs"` - PathHome string `json:"path_home"` + ClusterID string `json:"cluster_id"` + ClusterUUID string `json:"cluster_uuid"` + NodeUUID string `json:"node_uuid"` + + //PublishAddress string `json:"publish_address"` + //NodeName string `json:"node_name"` + //PathLogs string `json:"path_logs"` + //PathHome string `json:"path_home"` + //ClusterName string `json:"cluster_name"` //infini system assigned id - ClusterID string `json:"cluster_id"` - Updated int64 `json:"updated"` + Updated int64 `json:"updated"` } func GetElasticLogFiles(ctx context.Context, instance *model.Instance, logsPath string) (interface{}, error) { @@ -255,8 +261,9 @@ func GetElasticLogFileContent(ctx context.Context, instance *model.Instance, bod } func (h *APIHandler) getLogFilesByNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + clusterID := ps.MustGetParameter("id") nodeID := ps.MustGetParameter("node_id") - inst, pathLogs, err := getAgentByNodeID(nodeID) + inst, pathLogs, err := getAgentByNodeID(clusterID, nodeID) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) log.Error(err) @@ -283,8 +290,9 @@ func (h *APIHandler) getLogFilesByNode(w http.ResponseWriter, req *http.Request, } func (h *APIHandler) getLogFileContent(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + clusterID := ps.MustGetParameter("id") nodeID := ps.MustGetParameter("node_id") - inst, pathLogs, err := getAgentByNodeID(nodeID) + inst, pathLogs, err := getAgentByNodeID(clusterID, nodeID) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) log.Error(err) @@ -318,12 +326,13 @@ func (h *APIHandler) getLogFileContent(w http.ResponseWriter, req *http.Request, } //instance, pathLogs -func getAgentByNodeID(nodeID string) (*model.Instance, string, error) { +func getAgentByNodeID(clusterID, nodeID string) (*model.Instance, string, error) { q := orm.Query{ Size: 1000, Conds: orm.And(orm.Eq("metadata.category", "node_settings"), orm.Eq("metadata.name", "agent"), + orm.Eq("payload.cluster_id", clusterID), orm.Eq("payload.node_uuid", nodeID), ), } @@ -333,17 +342,25 @@ func getAgentByNodeID(nodeID string) (*model.Instance, string, error) { return nil, "", err } + nodeInfo,err:=metadata.GetNodeConfig(clusterID, nodeID) + if err!=nil||nodeInfo==nil{ + log.Error("node info is nil") + return nil, "", err + } + + pathLogs := nodeInfo.Payload.NodeInfo.GetPathLogs() + for _, row := range result.Result { v, ok := row.(map[string]interface{}) if ok { - pathLogs := "" - payload, ok := v["payload"] - if ok { - payloadMap, ok := payload.(map[string]interface{}) - if ok { - pathLogs = util.ToString(payloadMap["path_logs"]) - } - } + //payload, ok := v["payload"] + //if ok { + // payloadMap, ok := payload.(map[string]interface{}) + // if ok { + // pathLogs = util.ToString(payloadMap["path_logs"]) + // } + //} + x, ok := v["metadata"] if ok { @@ -417,6 +434,15 @@ func (h *APIHandler) discoveryESNodesInfo(w http.ResponseWriter, req *http.Reque for _, clusterID := range clusterInfo.ClusterIDs { meta := elastic.GetMetadata(clusterID) if meta != nil { + + states,err:=elastic.GetClient(clusterID).GetClusterState() + if err!=nil||states==nil{ + log.Error(err) + continue + } + + clusterUUID:=states.ClusterUUID + if meta.Config.AgentCredentialID != "" { auth, err := common.GetAgentBasicAuth(meta.Config) if err != nil { @@ -438,8 +464,34 @@ func (h *APIHandler) discoveryESNodesInfo(w http.ResponseWriter, req *http.Reque } if success { - log.Debug("connect to es node success:", nodeHost,", pid: ",node.PID) + log.Debug("connect to es node success:", nodeHost, ", pid: ", node.PID) discoveredPIDs[node.PID] = nodeInfo + + + if nodeInfo.ClusterInfo.ClusterUUID!=clusterUUID{ + log.Error("cluster uuid not match, cluster id: ", clusterID, ", cluster uuid: ", clusterUUID, ", node cluster uuid: ", nodeInfo.ClusterInfo.ClusterUUID) + continue + } + + //enroll this node + item := BindingItem{ + ClusterID: clusterID, + ClusterUUID: nodeInfo.ClusterInfo.ClusterUUID, + //ClusterName: nodeInfo.ClusterInfo.ClusterName, + //NodeName: nodeInfo.NodeInfo.Name, + NodeUUID: nodeInfo.NodeUUID, + //PathHome: nodeInfo.NodeInfo., + } + + settings := NewNodeAgentSettings(instance.ID, &item) + err = orm.Update(&orm.Context{ + Refresh: "wait_for", + }, settings) + + if err == nil { + nodeInfo.ClusterID = clusterID + nodeInfo.Enrolled = true + } break } } @@ -542,14 +594,15 @@ func NewNodeAgentSettings(instanceID string, item *BindingItem) *model.Setting { } settings.Payload = util.MapStr{ - "cluster_id": item.ClusterID, - "cluster_name": item.ClusterName, - "cluster_uuid": item.ClusterUUID, - "node_uuid": item.NodeUUID, - "publish_address": item.PublishAddress, - "node_name": item.NodeName, - "path_home": item.PathHome, - "path_logs": item.PathLogs, + "cluster_id": item.ClusterID, + "cluster_uuid": item.ClusterUUID, + "node_uuid": item.NodeUUID, + + //"cluster_name": item.ClusterName, + //"publish_address": item.PublishAddress, + //"node_name": item.NodeName, + //"path_home": item.PathHome, + //"path_logs": item.PathLogs, } return &settings diff --git a/modules/agent/api/init.go b/modules/agent/api/init.go index 5cb4328a..48cb6b8a 100644 --- a/modules/agent/api/init.go +++ b/modules/agent/api/init.go @@ -28,8 +28,6 @@ func Init() { api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_enroll", handler.RequirePermission(handler.enrollESNode, enum.PermissionAgentInstanceWrite)) api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_revoke", handler.RequirePermission(handler.revokeESNode, enum.PermissionAgentInstanceWrite)) - //api.HandleAPIMethod(api.POST, "/auto_associate", handler.RequirePermission(handler.autoAssociateESNode, enum.PermissionAgentInstanceWrite)) - //get elasticsearch node logs, direct fetch or via stored logs(TODO) api.HandleAPIMethod(api.GET, "/elasticsearch/:id/node/:node_id/logs/_list", handler.RequirePermission(handler.getLogFilesByNode, enum.PermissionAgentInstanceRead)) api.HandleAPIMethod(api.POST, "/elasticsearch/:id/node/:node_id/logs/_read", handler.RequirePermission(handler.getLogFileContent, enum.PermissionAgentInstanceRead)) diff --git a/modules/agent/api/remote_config.go b/modules/agent/api/remote_config.go index e8f903f6..3f8c663f 100644 --- a/modules/agent/api/remote_config.go +++ b/modules/agent/api/remote_config.go @@ -15,6 +15,7 @@ import ( "infini.sh/framework/core/orm" "infini.sh/framework/core/util" common2 "infini.sh/framework/modules/elastic/common" + metadata2 "infini.sh/framework/modules/elastic/metadata" "infini.sh/framework/plugins/managed/common" "time" ) @@ -192,12 +193,22 @@ func getAgentIngestConfigs(instance string, items map[string]BindingItem) (strin } } - if v.PublishAddress==""{ + nodeInfo,err:=metadata2.GetNodeConfig(v.ClusterID,v.NodeUUID) + if err!=nil{ + log.Error(err) + continue + } + + publishAddress:=nodeInfo.Payload.NodeInfo.GetHttpPublishHost() + + if publishAddress==""{ log.Errorf("publish address is empty: %v",v.NodeUUID) continue } - nodeEndPoint := metadata.PrepareEndpoint(v.PublishAddress) + nodeEndPoint := metadata.PrepareEndpoint(publishAddress) + + pathLogs:=nodeInfo.Payload.NodeInfo.GetPathLogs() if v.Updated > latestVersion { latestVersion = v.Updated @@ -217,7 +228,7 @@ func getAgentIngestConfigs(instance string, items map[string]BindingItem) (strin "CLUSTER_LEVEL_TASKS_ENABLED: %v\n "+ "NODE_LEVEL_TASKS_ENABLED: %v\n "+ "NODE_LOGS_PATH: \"%v\"\n\n\n", taskID, taskID, - v.ClusterID,v.ClusterUUID,v.NodeUUID, nodeEndPoint, username, password, clusterLevelEnabled, nodeLevelEnabled, v.PathLogs))) + v.ClusterID,v.ClusterUUID,v.NodeUUID, nodeEndPoint, username, password, clusterLevelEnabled, nodeLevelEnabled, pathLogs))) } hash := util.MD5digest(buffer.String()) From 287d53111bf9777cc55fff59f4189fb07e610fc3 Mon Sep 17 00:00:00 2001 From: medcl Date: Sun, 29 Oct 2023 10:18:29 +0800 Subject: [PATCH 36/36] update auto enroll api --- modules/agent/api/elasticsearch.go | 44 ++++++++++-------------------- modules/agent/api/init.go | 3 +- 2 files changed, 16 insertions(+), 31 deletions(-) diff --git a/modules/agent/api/elasticsearch.go b/modules/agent/api/elasticsearch.go index 0def2e83..168608ed 100644 --- a/modules/agent/api/elasticsearch.go +++ b/modules/agent/api/elasticsearch.go @@ -52,12 +52,8 @@ func GetEnrolledNodesByAgent(instance *model.Instance) (map[string]BindingItem, if ok { item := BindingItem{} item.ClusterID = util.ToString(f["cluster_id"]) - //item.ClusterName = util.ToString(f["cluster_name"]) + item.ClusterUUID = util.ToString(f["cluster_uuid"]) - //item.PublishAddress = util.ToString(f["publish_address"]) - //item.NodeName = util.ToString(f["node_name"]) - //item.PathHome = util.ToString(f["path_home"]) - //item.PathLogs = util.ToString(f["path_logs"]) item.NodeUUID = nodeID t, ok := v["updated"] @@ -196,17 +192,12 @@ func GetElasticsearchNodesViaAgent(ctx context.Context, instance *model.Instance } type BindingItem struct { + //infini system assigned id ClusterID string `json:"cluster_id"` + ClusterUUID string `json:"cluster_uuid"` NodeUUID string `json:"node_uuid"` - //PublishAddress string `json:"publish_address"` - //NodeName string `json:"node_name"` - //PathLogs string `json:"path_logs"` - //PathHome string `json:"path_home"` - //ClusterName string `json:"cluster_name"` - - //infini system assigned id Updated int64 `json:"updated"` } @@ -353,14 +344,6 @@ func getAgentByNodeID(clusterID, nodeID string) (*model.Instance, string, error) for _, row := range result.Result { v, ok := row.(map[string]interface{}) if ok { - //payload, ok := v["payload"] - //if ok { - // payloadMap, ok := payload.(map[string]interface{}) - // if ok { - // pathLogs = util.ToString(payloadMap["path_logs"]) - // } - //} - x, ok := v["metadata"] if ok { @@ -394,7 +377,17 @@ type ClusterInfo struct { } func (h *APIHandler) autoEnrollESNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - //TODO + + //{"cluster_id":["infini_default_system_cluster"]} + + //get instances + //get all unknown nodes + //check each process with cluster id + + //send this to background task + + + h.WriteAckOKJSON(w) } func (h *APIHandler) discoveryESNodesInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { @@ -477,10 +470,7 @@ func (h *APIHandler) discoveryESNodesInfo(w http.ResponseWriter, req *http.Reque item := BindingItem{ ClusterID: clusterID, ClusterUUID: nodeInfo.ClusterInfo.ClusterUUID, - //ClusterName: nodeInfo.ClusterInfo.ClusterName, - //NodeName: nodeInfo.NodeInfo.Name, NodeUUID: nodeInfo.NodeUUID, - //PathHome: nodeInfo.NodeInfo., } settings := NewNodeAgentSettings(instance.ID, &item) @@ -597,12 +587,6 @@ func NewNodeAgentSettings(instanceID string, item *BindingItem) *model.Setting { "cluster_id": item.ClusterID, "cluster_uuid": item.ClusterUUID, "node_uuid": item.NodeUUID, - - //"cluster_name": item.ClusterName, - //"publish_address": item.PublishAddress, - //"node_name": item.NodeName, - //"path_home": item.PathHome, - //"path_logs": item.PathLogs, } return &settings diff --git a/modules/agent/api/init.go b/modules/agent/api/init.go index 48cb6b8a..b3b19ce0 100644 --- a/modules/agent/api/init.go +++ b/modules/agent/api/init.go @@ -24,10 +24,11 @@ func Init() { //bind agent with nodes api.HandleAPIMethod(api.GET, "/instance/:instance_id/node/_discovery", handler.RequirePermission(handler.discoveryESNodesInfo, enum.PermissionAgentInstanceRead)) api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_discovery", handler.RequirePermission(handler.discoveryESNodesInfo, enum.PermissionAgentInstanceRead)) - api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_auto_enroll", handler.RequirePermission(handler.autoEnrollESNode, enum.PermissionAgentInstanceWrite)) api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_enroll", handler.RequirePermission(handler.enrollESNode, enum.PermissionAgentInstanceWrite)) api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_revoke", handler.RequirePermission(handler.revokeESNode, enum.PermissionAgentInstanceWrite)) + api.HandleAPIMethod(api.POST, "/instance/node/_auto_enroll", handler.RequirePermission(handler.autoEnrollESNode, enum.PermissionAgentInstanceWrite)) + //get elasticsearch node logs, direct fetch or via stored logs(TODO) api.HandleAPIMethod(api.GET, "/elasticsearch/:id/node/:node_id/logs/_list", handler.RequirePermission(handler.getLogFilesByNode, enum.PermissionAgentInstanceRead)) api.HandleAPIMethod(api.POST, "/elasticsearch/:id/node/:node_id/logs/_read", handler.RequirePermission(handler.getLogFileContent, enum.PermissionAgentInstanceRead))