floraachy
/
console
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

temp commit

This commit is contained in:
medcl 2023-09-16 17:45:33 +08:00
parent 1b0022a86d
commit 3673791467
26 changed files with 1101 additions and 2719 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
config/ingest_config.tpl Normal file
View File

@ -0,0 +1,90 @@
elasticsearch:
- name: $[[INGEST_CLUSTER_ID]]
enabled: true
endpoints: $[[INGEST_CLUSTER_ENDPOINT]]
discovery:
enabled: false
basic_auth:
username: $[[INGEST_CLUSTER_USERNAME]]
password: $[[keystore.ingest_cluster_password]]
metrics:
enabled: true
queue: metrics
network:
enabled: true
summary: true
details: true
memory:
metrics:
- swap
- memory
disk:
metrics:
- iops
- usage
cpu:
metrics:
- idle
- system
- user
- iowait
- load
instance:
enabled: true
elastic:
availability_check:
enabled: false
pipeline:
- name: merge_logs
auto_start: true
keep_running: true
processor:
- indexing_merge:
index_name: ".infini_logs"
elasticsearch: "$[[INGEST_CLUSTER_ID]]"
input_queue: "logs"
idle_timeout_in_seconds: 10
output_queue:
name: "merged_requests"
worker_size: 1
bulk_size_in_mb: 5
- name: merge_metrics
auto_start: true
keep_running: true
processor:
- indexing_merge:
elasticsearch: "$[[INGEST_CLUSTER_ID]]"
index_name: ".infini_metrics"
input_queue: "metrics"
output_queue:
name: "merged_requests"
worker_size: 1
bulk_size_in_mb: 5
- name: ingest_merged_requests
auto_start: true
keep_running: true
processor:
- bulk_indexing:
max_worker_size: 1
bulk:
batch_size_in_mb: 5
batch_size_in_docs: 5000
max_retry_times: 0
invalid_queue: ""
response_handle:
include_index_stats: false
include_action_stats: false
output_bulk_stats: false
include_error_details: false
save_error_results: false
save_success_results: false
save_busy_results: false
consumer:
fetch_max_messages: 5
queues:
type: indexing_merge
when:
cluster_available: ["$[[INGEST_CLUSTER_ID]]"]

22
config/install_agent.tpl
View File

@ -220,6 +220,23 @@ path.data: data
path.logs: log path.logs: log
path.configs: config path.configs: config
resource_limit.cpu.max_num_of_cpus: 1
resource_limit.memory.max_in_bytes: 533708800
stats:
include_storage_stats_in_api: false
disk_queue:
max_msg_size: 20485760
max_bytes_per_file: 20485760
max_used_bytes: 524288000
retention.max_num_of_local_files: 1
compress:
idle_threshold: 0
num_of_files_decompress_ahead: 0
segment:
enabled: true
api: api:
enabled: true enabled: true
tls: tls:
@ -232,11 +249,12 @@ api:
binding: \$[[env.API_BINDING]] binding: \$[[env.API_BINDING]]
badger: badger:
value_threshold: 1024
mem_table_size: 1048576
value_log_max_entries: 1000000 value_log_max_entries: 1000000
value_log_file_size: 104857600 value_log_file_size: 104857600
value_threshold: 1024
agent: node:
major_ip_pattern: ".*" major_ip_pattern: ".*"
EOF EOF
} }

34
config/system_config.tpl
View File

@ -7,7 +7,7 @@ elasticsearch:
enabled: true enabled: true
monitored: true monitored: true
reserved: true reserved: true
endpoint: $[[CLUSTER_ENDPINT]] endpoint: $[[CLUSTER_ENDPOINT]]
discovery: discovery:
enabled: false enabled: false
basic_auth: basic_auth:
@ -17,7 +17,7 @@ elasticsearch:
elastic.elasticsearch: $[[CLUSTER_ID]] elastic.elasticsearch: $[[CLUSTER_ID]]
pipeline: pipeline:
- name: indexing_merge - name: merge_metrics
auto_start: true auto_start: true
keep_running: true keep_running: true
processor: processor:
@ -31,22 +31,7 @@ pipeline:
tag: "metrics" tag: "metrics"
worker_size: 1 worker_size: 1
bulk_size_in_mb: 5 bulk_size_in_mb: 5
- name: consume-metrics_requests
auto_start: true
keep_running: true
processor:
- bulk_indexing:
bulk:
compress: true
batch_size_in_mb: 5
batch_size_in_docs: 5000
consumer:
fetch_max_messages: 100
queues:
type: indexing_merge
tag: "metrics"
when:
cluster_available: ["$[[CLUSTER_ID]]"]
- name: metadata_ingest - name: metadata_ingest
auto_start: true auto_start: true
keep_running: true keep_running: true
@ -91,7 +76,7 @@ pipeline:
when: when:
cluster_available: ["$[[CLUSTER_ID]]"] cluster_available: ["$[[CLUSTER_ID]]"]
- name: logging_indexing_merge - name: merge_logging
auto_start: true auto_start: true
keep_running: true keep_running: true
processor: processor:
@ -106,19 +91,22 @@ pipeline:
tag: "request_logging" tag: "request_logging"
worker_size: 1 worker_size: 1
bulk_size_in_kb: 1 bulk_size_in_kb: 1
- name: consume-logging_requests
- name: ingest_merged_requests
auto_start: true auto_start: true
keep_running: true keep_running: true
retry_delay_in_ms: 5000
max_running_in_ms: 30000
processor: processor:
- bulk_indexing: - bulk_indexing:
idle_timeout_in_seconds: 5
bulk: bulk:
compress: true compress: true
batch_size_in_mb: 1 batch_size_in_mb: 10
batch_size_in_docs: 1 batch_size_in_docs: 1000
consumer: consumer:
fetch_max_messages: 100 fetch_max_messages: 100
queues: queues:
type: indexing_merge type: indexing_merge
tag: "request_logging"
when: when:
cluster_available: ["$[[CLUSTER_ID]]"] cluster_available: ["$[[CLUSTER_ID]]"]

84
config/task_config.tpl Normal file
View File

@ -0,0 +1,84 @@
elasticsearch:
- id: $[[CLUSTER_ID]]
name: $[[CLUSTER_ID]]
enabled: true
endpoint: $[[CLUSTER_ENDPOINT]]
discovery:
enabled: false
basic_auth:
username: $[[CLUSTER_USERNAME]]
password: $[[keystore.$[[CLUSTER_ID]]_password]]
pipeline:
#clsuter level metrics
- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]]
enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]]
keep_running: true
singleton: true
name: collect_$[[CLUSTER_ID]]_es_cluster_stats
retry_delay_in_ms: 10000
processor:
- es_cluster_stats:
elasticsearch: $[[CLUSTER_ID]]
labels:
cluster_id: $[[CLUSTER_ID]]
when:
cluster_available: ["$[[CLUSTER_ID]]"]
- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]]
enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]]
keep_running: true
singleton: true
name: collect_$[[CLUSTER_ID]]_es_index_stats
retry_delay_in_ms: 10000
processor:
- es_index_stats:
elasticsearch: $[[CLUSTER_ID]]
labels:
cluster_id: $[[CLUSTER_ID]]
when:
cluster_available: ["$[[CLUSTER_ID]]"]
- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]]
enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]]
keep_running: true
singleton: true
name: collect_$[[CLUSTER_ID]]_es_cluster_health
retry_delay_in_ms: 10000
processor:
- es_cluster_health:
elasticsearch: $[[CLUSTER_ID]]
labels:
cluster_id: $[[CLUSTER_ID]]
when:
cluster_available: ["$[[CLUSTER_ID]]"]
#node level metrics
- auto_start: $[[NODE_LEVEL_TASKS_ENABLED]]
enabled: $[[NODE_LEVEL_TASKS_ENABLED]]
keep_running: true
name: collect_$[[CLUSTER_ID]]_es_node_stats
retry_delay_in_ms: 10000
processor:
- es_node_stats:
elasticsearch: $[[CLUSTER_ID]]
labels:
cluster_id: $[[CLUSTER_ID]]
when:
cluster_available: ["$[[CLUSTER_ID]]"]
#node logs
- auto_start: $[[NODE_LEVEL_TASKS_ENABLED]]
enabled: $[[NODE_LEVEL_TASKS_ENABLED]]
keep_running: true
name: collect_$[[CLUSTER_ID]]_es_logs
retry_delay_in_ms: 10000
processor:
- es_logs_processor:
elasticsearch: $[[CLUSTER_ID]]
labels:
cluster_id: $[[CLUSTER_ID]]
logs_path: $[[NODE_LOGS_PATH]]
queue_name: logs
when:
cluster_available: ["$[[CLUSTER_ID]]"]

1
console.yml
View File

@ -52,7 +52,6 @@ elastic:
metrics: metrics:
enabled: true enabled: true
major_ip_pattern: "192.*"
queue: metrics queue: metrics
elasticsearch: elasticsearch:
enabled: true enabled: true

12
main.go
View File

@ -5,6 +5,7 @@ import (
"errors" "errors"
_ "expvar" _ "expvar"
"infini.sh/console/plugin/api/email" "infini.sh/console/plugin/api/email"
model2 "infini.sh/framework/core/model"
_ "time/tzdata" _ "time/tzdata"
log "github.com/cihub/seelog" log "github.com/cihub/seelog"
@ -12,7 +13,6 @@ import (
"infini.sh/console/model" "infini.sh/console/model"
"infini.sh/console/model/alerting" "infini.sh/console/model/alerting"
"infini.sh/console/model/insight" "infini.sh/console/model/insight"
"infini.sh/console/modules/agent"
_ "infini.sh/console/plugin" _ "infini.sh/console/plugin"
setup1 "infini.sh/console/plugin/setup" setup1 "infini.sh/console/plugin/setup"
alerting2 "infini.sh/console/service/alerting" alerting2 "infini.sh/console/service/alerting"
@ -37,6 +37,7 @@ import (
_ "infini.sh/framework/plugins" _ "infini.sh/framework/plugins"
api2 "infini.sh/gateway/api" api2 "infini.sh/gateway/api"
_ "infini.sh/gateway/proxy" _ "infini.sh/gateway/proxy"
_ "infini.sh/framework/plugins/managed"
) )
var appConfig *config.AppConfig var appConfig *config.AppConfig
@ -70,7 +71,6 @@ func main() {
modules = append(modules, module.ModuleItem{Value: &task.TaskModule{}, Priority: 1}) modules = append(modules, module.ModuleItem{Value: &task.TaskModule{}, Priority: 1})
modules = append(modules, module.ModuleItem{Value: &metrics.MetricsModule{}, Priority: 1}) modules = append(modules, module.ModuleItem{Value: &metrics.MetricsModule{}, Priority: 1})
modules = append(modules, module.ModuleItem{Value: &security.Module{}, Priority: 1}) modules = append(modules, module.ModuleItem{Value: &security.Module{}, Priority: 1})
modules = append(modules, module.ModuleItem{Value: &agent.AgentModule{}, Priority: 100})
uiModule := &ui.UIModule{} uiModule := &ui.UIModule{}
@ -122,11 +122,11 @@ func main() {
elastic2.InitTemplate(false) elastic2.InitTemplate(false)
//orm.RegisterSchemaWithIndexName(model.Dict{}, "dict") //orm.RegisterSchema(model.Dict{}, "dict")
orm.RegisterSchemaWithIndexName(elastic.View{}, "view") orm.RegisterSchemaWithIndexName(elastic.View{}, "view")
orm.RegisterSchemaWithIndexName(elastic.CommonCommand{}, "commands") orm.RegisterSchemaWithIndexName(elastic.CommonCommand{}, "commands")
//orm.RegisterSchemaWithIndexName(elastic.TraceTemplate{}, "trace-template") //orm.RegisterSchema(elastic.TraceTemplate{}, "trace-template")
orm.RegisterSchemaWithIndexName(model.Instance{}, "instance") //orm.RegisterSchema(model.Instance{}, "instance")
orm.RegisterSchemaWithIndexName(alerting.Rule{}, "alert-rule") orm.RegisterSchemaWithIndexName(alerting.Rule{}, "alert-rule")
orm.RegisterSchemaWithIndexName(alerting.Alert{}, "alert-history") orm.RegisterSchemaWithIndexName(alerting.Alert{}, "alert-history")
orm.RegisterSchemaWithIndexName(alerting.AlertMessage{}, "alert-message") orm.RegisterSchemaWithIndexName(alerting.AlertMessage{}, "alert-message")
@ -138,6 +138,8 @@ func main() {
orm.RegisterSchemaWithIndexName(model.Layout{}, "layout") orm.RegisterSchemaWithIndexName(model.Layout{}, "layout")
orm.RegisterSchemaWithIndexName(model.Notification{}, "notification") orm.RegisterSchemaWithIndexName(model.Notification{}, "notification")
orm.RegisterSchemaWithIndexName(model.EmailServer{}, "email-server") orm.RegisterSchemaWithIndexName(model.EmailServer{}, "email-server")
orm.RegisterSchemaWithIndexName(model2.Instance{}, "instance")
api.RegisterSchema() api.RegisterSchema()
if global.Env().SetupRequired() { if global.Env().SetupRequired() {

44
model/instance.go
View File

@ -11,26 +11,15 @@ import (
"net/http" "net/http"
"time" "time"
"infini.sh/framework/core/agent" "infini.sh/framework/core/model"
"infini.sh/framework/core/orm"
"infini.sh/framework/core/util" "infini.sh/framework/core/util"
"infini.sh/framework/modules/pipeline" "infini.sh/framework/modules/pipeline"
) )
type TaskWorker struct {
type Instance struct { model.Instance
orm.ORMObjectBase
//InstanceID string `json:"instance_id,omitempty" elastic_mapping:"instance_id: { type: keyword }"`
Name string `json:"name,omitempty" elastic_mapping:"name:{type:keyword,fields:{text: {type: text}}}"`
Endpoint string `json:"endpoint,omitempty" elastic_mapping:"endpoint: { type: keyword }"`
Version map[string]interface{} `json:"version,omitempty" elastic_mapping:"version: { type: object }"`
BasicAuth agent.BasicAuth `config:"basic_auth" json:"basic_auth,omitempty" elastic_mapping:"basic_auth:{type:object}"`
Owner string `json:"owner,omitempty" config:"owner" elastic_mapping:"owner:{type:keyword}"`
Tags []string `json:"tags,omitempty"`
Description string `json:"description,omitempty" config:"description" elastic_mapping:"description:{type:keyword}"`
} }
func (inst *Instance) CreatePipeline(body []byte) error { func (inst *TaskWorker) CreatePipeline(body []byte) error {
req := &util.Request{ req := &util.Request{
Method: http.MethodPost, Method: http.MethodPost,
Body: body, Body: body,
@ -39,7 +28,7 @@ func (inst *Instance) CreatePipeline(body []byte) error {
return inst.doRequest(req, nil) return inst.doRequest(req, nil)
} }
func (inst *Instance) StopPipeline(ctx context.Context, pipelineID string) error { func (inst *TaskWorker) StopPipeline(ctx context.Context, pipelineID string) error {
req := &util.Request{ req := &util.Request{
Method: http.MethodPost, Method: http.MethodPost,
Url: fmt.Sprintf("%s/pipeline/task/%s/_stop", inst.Endpoint, pipelineID), Url: fmt.Sprintf("%s/pipeline/task/%s/_stop", inst.Endpoint, pipelineID),
@ -48,13 +37,13 @@ func (inst *Instance) StopPipeline(ctx context.Context, pipelineID string) error
return inst.doRequest(req, nil) return inst.doRequest(req, nil)
} }
func (inst *Instance) StopPipelineWithTimeout(pipelineID string, duration time.Duration) error { func (inst *TaskWorker) StopPipelineWithTimeout(pipelineID string, duration time.Duration) error {
ctx, cancel := context.WithTimeout(context.Background(), duration) ctx, cancel := context.WithTimeout(context.Background(), duration)
defer cancel() defer cancel()
return inst.StopPipeline(ctx, pipelineID) return inst.StopPipeline(ctx, pipelineID)
} }
func (inst *Instance) StartPipeline(pipelineID string) error { func (inst *TaskWorker) StartPipeline(pipelineID string) error {
req := &util.Request{ req := &util.Request{
Method: http.MethodPost, Method: http.MethodPost,
Url: fmt.Sprintf("%s/pipeline/task/%s/_start", inst.Endpoint, pipelineID), Url: fmt.Sprintf("%s/pipeline/task/%s/_start", inst.Endpoint, pipelineID),
@ -62,7 +51,7 @@ func (inst *Instance) StartPipeline(pipelineID string) error {
return inst.doRequest(req, nil) return inst.doRequest(req, nil)
} }
func (inst *Instance) DeletePipeline(pipelineID string) error { func (inst *TaskWorker) DeletePipeline(pipelineID string) error {
req := &util.Request{ req := &util.Request{
Method: http.MethodDelete, Method: http.MethodDelete,
Url: fmt.Sprintf("%s/pipeline/task/%s", inst.Endpoint, pipelineID), Url: fmt.Sprintf("%s/pipeline/task/%s", inst.Endpoint, pipelineID),
@ -70,7 +59,7 @@ func (inst *Instance) DeletePipeline(pipelineID string) error {
return inst.doRequest(req, nil) return inst.doRequest(req, nil)
} }
func (inst *Instance) GetPipeline(pipelineID string) (*pipeline.PipelineStatus, error) { func (inst *TaskWorker) GetPipeline(pipelineID string) (*pipeline.PipelineStatus, error) {
if pipelineID == "" { if pipelineID == "" {
return nil, errors.New("invalid pipelineID") return nil, errors.New("invalid pipelineID")
} }
@ -89,7 +78,7 @@ func (inst *Instance) GetPipeline(pipelineID string) (*pipeline.PipelineStatus,
return &res, nil return &res, nil
} }
func (inst *Instance) GetPipelinesByIDs(pipelineIDs []string) (pipeline.GetPipelinesResponse, error) { func (inst *TaskWorker) GetPipelinesByIDs(pipelineIDs []string) (pipeline.GetPipelinesResponse, error) {
body := util.MustToJSONBytes(util.MapStr{ body := util.MustToJSONBytes(util.MapStr{
"ids": pipelineIDs, "ids": pipelineIDs,
}) })
@ -106,7 +95,7 @@ func (inst *Instance) GetPipelinesByIDs(pipelineIDs []string) (pipeline.GetPipel
return res, err return res, err
} }
func (inst *Instance) DeleteQueueBySelector(selector util.MapStr) error { func (inst *TaskWorker) DeleteQueueBySelector(selector util.MapStr) error {
req := &util.Request{ req := &util.Request{
Method: http.MethodDelete, Method: http.MethodDelete,
Url: fmt.Sprintf("%s/queue/_search", inst.Endpoint), Url: fmt.Sprintf("%s/queue/_search", inst.Endpoint),
@ -117,7 +106,7 @@ func (inst *Instance) DeleteQueueBySelector(selector util.MapStr) error {
return inst.doRequest(req, nil) return inst.doRequest(req, nil)
} }
func (inst *Instance) DeleteQueueConsumersBySelector(selector util.MapStr) error { func (inst *TaskWorker) DeleteQueueConsumersBySelector(selector util.MapStr) error {
req := &util.Request{ req := &util.Request{
Method: http.MethodDelete, Method: http.MethodDelete,
Url: fmt.Sprintf("%s/queue/consumer/_search", inst.Endpoint), Url: fmt.Sprintf("%s/queue/consumer/_search", inst.Endpoint),
@ -128,21 +117,22 @@ func (inst *Instance) DeleteQueueConsumersBySelector(selector util.MapStr) error
return inst.doRequest(req, nil) return inst.doRequest(req, nil)
} }
func (inst *Instance) TryConnect(ctx context.Context) error { func (inst *TaskWorker) TryConnect(ctx context.Context) error {
req := &util.Request{ req := &util.Request{
Method: http.MethodGet, Method: http.MethodGet,
Url: fmt.Sprintf("%s/_framework/api/_info", inst.Endpoint), Url: fmt.Sprintf("%s/_info", inst.Endpoint),
Context: ctx, Context: ctx,
} }
return inst.doRequest(req, nil) return inst.doRequest(req, nil)
} }
func (inst *Instance) TryConnectWithTimeout(duration time.Duration) error {
func (inst *TaskWorker) TryConnectWithTimeout(duration time.Duration) error {
ctx, cancel := context.WithTimeout(context.Background(), duration) ctx, cancel := context.WithTimeout(context.Background(), duration)
defer cancel() defer cancel()
return inst.TryConnect(ctx) return inst.TryConnect(ctx)
} }
func (inst *Instance) doRequest(req *util.Request, resBody interface{}) error { func (inst *TaskWorker) doRequest(req *util.Request, resBody interface{}) error {
req.SetBasicAuth(inst.BasicAuth.Username, inst.BasicAuth.Password) req.SetBasicAuth(inst.BasicAuth.Username, inst.BasicAuth.Password)
result, err := util.ExecuteRequest(req) result, err := util.ExecuteRequest(req)
if err != nil { if err != nil {

158
modules/agent/agent.go
View File

@ -1,158 +0,0 @@
/* Copyright © INFINI Ltd. All rights reserved.
* Web: https://infinilabs.com
* Email: hello#infini.ltd */
package agent
import (
"fmt"
log "github.com/cihub/seelog"
"infini.sh/console/modules/agent/api"
"infini.sh/console/modules/agent/client"
"infini.sh/console/modules/agent/common"
"infini.sh/console/modules/agent/model"
"infini.sh/console/modules/agent/state"
"infini.sh/framework/core/agent"
"infini.sh/framework/core/credential"
"infini.sh/framework/core/elastic"
"infini.sh/framework/core/env"
"infini.sh/framework/core/host"
"infini.sh/framework/core/kv"
"infini.sh/framework/core/orm"
"infini.sh/framework/core/util"
"time"
)
func (module *AgentModule) Name() string {
return "agent"
}
func (module *AgentModule) Setup() {
module.AgentConfig.Enabled = true
module.AgentConfig.StateManager.Enabled = true
exists, err := env.ParseConfig("agent", &module.AgentConfig)
if exists && err != nil {
panic(err)
}
if module.AgentConfig.Enabled {
api.Init()
}
}
func (module *AgentModule) Start() error {
if !module.AgentConfig.Enabled {
return nil
}
orm.RegisterSchemaWithIndexName(agent.Instance{}, "agent")
orm.RegisterSchemaWithIndexName(agent.ESNodeInfo{}, "agent-node")
orm.RegisterSchemaWithIndexName(host.HostInfo{}, "host")
orm.RegisterSchemaWithIndexName(agent.Setting{}, "agent-setting")
var (
executor client.Executor
err error
caFile string
caKey string
)
if module.AgentConfig.Setup != nil {
caFile = module.AgentConfig.Setup.CACertFile
caKey = module.AgentConfig.Setup.CAKeyFile
}
if caFile == "" && caKey == "" {
caFile, caKey, err = common.GetOrInitDefaultCaCerts()
if err != nil {
panic(err)
}
}
executor, err = client.NewMTLSExecutor(caFile, caKey)
if err != nil {
panic(err)
}
agClient := &client.Client{
Executor: executor,
}
client.RegisterClient(agClient)
if module.AgentConfig.StateManager.Enabled {
onlineAgentIDs, err := common.GetLatestOnlineAgentIDs(nil, 60)
if err != nil {
log.Error(err)
}
agents, err := common.LoadAgentsFromES("")
if err != nil {
log.Error(err)
}
agentIds := map[string]string{}
for _, ag := range agents {
if _, ok := onlineAgentIDs[ag.ID]; ok {
agentIds[ag.ID] = "online"
}
}
credential.RegisterChangeEvent(func(cred *credential.Credential) {
var effectsClusterIDs []string
elastic.WalkConfigs(func(key, value interface{}) bool {
if cfg, ok := value.(*elastic.ElasticsearchConfig); ok {
if cfg.CredentialID == cred.ID {
effectsClusterIDs = append(effectsClusterIDs, cfg.ID)
}
}
return true
})
if len(effectsClusterIDs) > 0 {
queryDsl := util.MapStr{
"query": util.MapStr{
"bool": util.MapStr{
"must": []util.MapStr{
{
"terms": util.MapStr{
"metadata.labels.cluster_id": effectsClusterIDs,
},
},
},
},
},
"script": util.MapStr{
"source": fmt.Sprintf("ctx._source['updated'] = '%s'", time.Now().Format(time.RFC3339Nano)),
},
}
err = orm.UpdateBy(agent.Setting{}, util.MustToJSONBytes(queryDsl))
if err != nil {
log.Error(err)
}
}
//check ingest cluster credential
if module.AgentConfig.Setup != nil && module.AgentConfig.Setup.IngestClusterCredentialID == cred.ID {
agents, err = common.LoadAgentsFromES("")
if err != nil {
log.Error(err)
return
}
for _, ag := range agents {
err = kv.AddValue(model.KVAgentIngestConfigChanged, []byte(ag.ID), []byte("1"))
if err != nil {
log.Error(err)
}
}
}
})
sm := state.NewStateManager(time.Second*30, "agent_state", agentIds, agClient)
state.RegisterStateManager(sm)
go sm.LoopState()
}
return nil
}
func (module *AgentModule) Stop() error {
if !module.AgentConfig.Enabled {
return nil
}
log.Info("start to stop agent module")
if module.AgentConfig.StateManager.Enabled {
state.GetStateManager().Stop()
}
log.Info("agent module was stopped")
return nil
}
type AgentModule struct {
model.AgentConfig
}

453
modules/agent/api/host.go
View File

@ -4,236 +4,223 @@
package api package api
import ( //
"context" //func (h *APIHandler) enrollHost(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
"fmt" // var reqBody []struct {
log "github.com/cihub/seelog" // AgentID string `json:"agent_id"`
"infini.sh/console/modules/agent/state" // HostName string `json:"host_name"`
httprouter "infini.sh/framework/core/api/router" // IP string `json:"ip"`
"infini.sh/framework/core/host" // Source string `json:"source"`
"infini.sh/framework/core/orm" // OSName string `json:"os_name"`
"infini.sh/framework/core/util" // OSArch string `json:"os_arch"`
"net/http" // NodeID string `json:"node_uuid"`
"strings" // }
"time" // err := h.DecodeJSON(req, &reqBody)
) // if err != nil {
// log.Error(err)
func (h *APIHandler) enrollHost(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { // h.WriteError(w, err.Error(), http.StatusInternalServerError)
var reqBody []struct { // return
AgentID string `json:"agent_id"` // }
HostName string `json:"host_name"` // errors := util.MapStr{}
IP string `json:"ip"` // for i, hi := range reqBody {
Source string `json:"source"` // var (
OSName string `json:"os_name"` // hostInfo *host.HostInfo
OSArch string `json:"os_arch"` // )
NodeID string `json:"node_uuid"` // switch hi.Source {
} // case "agent":
err := h.DecodeJSON(req, &reqBody) // hostInfo, err = enrollHostFromAgent(hi.AgentID)
if err != nil { // if err != nil {
log.Error(err) // errors[hi.IP] = util.MapStr{
h.WriteError(w, err.Error(), http.StatusInternalServerError) // "error": err.Error(),
return // }
} // log.Error(err)
errors := util.MapStr{} // continue
for i, hi := range reqBody { // }
var ( // hostInfo.IP = hi.IP
hostInfo *host.HostInfo // hostInfo.AgentID = hi.AgentID
) // err = orm.Create(nil, hostInfo)
switch hi.Source { // if err != nil {
case "agent": // errors[hi.IP] = util.MapStr{
hostInfo, err = enrollHostFromAgent(hi.AgentID) // "error": err.Error(),
if err != nil { // }
errors[hi.IP] = util.MapStr{ // log.Error(err)
"error": err.Error(), // continue
} // }
log.Error(err) // case "es_node":
continue // hostInfo = &host.HostInfo{
} // IP: hi.IP,
hostInfo.IP = hi.IP // OSInfo: host.OS{
hostInfo.AgentID = hi.AgentID // Platform: hi.OSName,
err = orm.Create(nil, hostInfo) // KernelArch: hi.OSArch,
if err != nil { // },
errors[hi.IP] = util.MapStr{ // NodeID: hi.NodeID,
"error": err.Error(), // }
} // default:
log.Error(err) // errors[hi.IP] = util.MapStr{
continue // "error": fmt.Errorf("unkonow source type"),
} // }
case "es_node": // continue
hostInfo = &host.HostInfo{ // }
IP: hi.IP, // hostInfo.Timestamp = time.Now()
OSInfo: host.OS{ // var ctx *orm.Context
Platform: hi.OSName, // if i == len(reqBody) - 1 {
KernelArch: hi.OSArch, // ctx = &orm.Context{
}, // Refresh: "wait_for",
NodeID: hi.NodeID, // }
} // }
default: // hostInfo.OSInfo.Platform = strings.ToLower(hostInfo.OSInfo.Platform)
errors[hi.IP] = util.MapStr{ // err = orm.Create(ctx, hostInfo)
"error": fmt.Errorf("unkonow source type"), // if err != nil {
} // errors[hi.IP] = util.MapStr{
continue // "error": err.Error(),
} // }
hostInfo.Timestamp = time.Now() // log.Error(err)
var ctx *orm.Context // continue
if i == len(reqBody) - 1 { // }
ctx = &orm.Context{ // }
Refresh: "wait_for", // resBody := util.MapStr{
} // "success": true,
} // }
hostInfo.OSInfo.Platform = strings.ToLower(hostInfo.OSInfo.Platform) // if len(errors) > 0 {
err = orm.Create(ctx, hostInfo) // resBody["errors"] = errors
if err != nil { // resBody["success"] = false
errors[hi.IP] = util.MapStr{ // }
"error": err.Error(), //
} // h.WriteJSON(w, resBody, http.StatusOK)
log.Error(err) //}
continue //
} //func (h *APIHandler) deleteHost(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
} // hostID := ps.MustGetParameter("host_id")
resBody := util.MapStr{ // hostInfo, err := getHost(hostID)
"success": true, // if err != nil {
} // log.Error(err)
if len(errors) > 0 { // h.WriteError(w, err.Error(), http.StatusInternalServerError)
resBody["errors"] = errors // return
resBody["success"] = false // }
} // ctx := orm.Context{
// Refresh: "wait_for",
h.WriteJSON(w, resBody, http.StatusOK) // }
} // err = orm.Delete(&ctx, hostInfo)
// if err != nil {
func (h *APIHandler) deleteHost(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { // log.Error(err)
hostID := ps.MustGetParameter("host_id") // h.WriteError(w, err.Error(), http.StatusInternalServerError)
hostInfo, err := getHost(hostID) // return
if err != nil { // }
log.Error(err) // h.WriteDeletedOKJSON(w, hostID)
h.WriteError(w, err.Error(), http.StatusInternalServerError) //}
return //
} //func (h *APIHandler) GetHostAgentInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
ctx := orm.Context{ // hostID := ps.MustGetParameter("host_id")
Refresh: "wait_for", // hostInfo, err := getHost(hostID)
} // if err != nil {
err = orm.Delete(&ctx, hostInfo) // log.Error(err)
if err != nil { // h.WriteError(w, err.Error(), http.StatusInternalServerError)
log.Error(err) // return
h.WriteError(w, err.Error(), http.StatusInternalServerError) // }
return // if hostInfo.AgentID == "" {
} // h.WriteJSON(w, util.MapStr{}, http.StatusOK)
h.WriteDeletedOKJSON(w, hostID) // return
} // }
//
func (h *APIHandler) GetHostAgentInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { // sm := state.GetStateManager()
hostID := ps.MustGetParameter("host_id") // ag, err := sm.GetAgent(hostInfo.AgentID)
hostInfo, err := getHost(hostID) // if err != nil {
if err != nil { // log.Error(err)
log.Error(err) // h.WriteJSON(w, util.MapStr{}, http.StatusOK)
h.WriteError(w, err.Error(), http.StatusInternalServerError) // return
return // }
} // aversion, err := ag.GetVersion()
if hostInfo.AgentID == "" { // if err == nil {
h.WriteJSON(w, util.MapStr{}, http.StatusOK) // ag.Version = aversion
return // orm.Save(nil, ag)
} // }
// h.WriteJSON(w, util.MapStr{
sm := state.GetStateManager() // "host_id": hostID,
ag, err := sm.GetAgent(hostInfo.AgentID) // "agent_id": ag.ID,
if err != nil { // "version": ag.Version,
log.Error(err) // "status": hostInfo.AgentStatus,
h.WriteJSON(w, util.MapStr{}, http.StatusOK) // "endpoint": ag.GetEndpoint(),
return // }, http.StatusOK)
} //}
aversion, err := ag.GetVersion() //
if err == nil { //func getHost(hostID string) (*host.HostInfo, error){
ag.Version = aversion // hostInfo := &host.HostInfo{}
orm.Save(nil, ag) // hostInfo.ID = hostID
} // exists, err := orm.Get(hostInfo)
h.WriteJSON(w, util.MapStr{ // if err != nil {
"host_id": hostID, // return nil, fmt.Errorf("get host info error: %w", err)
"agent_id": ag.ID, // }
"version": ag.Version, // if !exists {
"status": hostInfo.AgentStatus, // return nil, fmt.Errorf("host [%s] not found", hostID)
"endpoint": ag.GetEndpoint(), // }
}, http.StatusOK) // return hostInfo, nil
} //}
//
func getHost(hostID string) (*host.HostInfo, error){ //func (h *APIHandler) GetHostElasticProcess(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
hostInfo := &host.HostInfo{} // hostID := ps.MustGetParameter("host_id")
hostInfo.ID = hostID // hostInfo := &host.HostInfo{}
exists, err := orm.Get(hostInfo) // hostInfo.ID = hostID
if err != nil { // exists, err := orm.Get(hostInfo)
return nil, fmt.Errorf("get host info error: %w", err) // if err != nil {
} // log.Error(err)
if !exists { // h.WriteError(w, err.Error(), http.StatusInternalServerError)
return nil, fmt.Errorf("host [%s] not found", hostID) // return
} // }
return hostInfo, nil // if !exists {
} // h.WriteError(w, fmt.Sprintf("host [%s] not found", hostID), http.StatusNotFound)
// return
func (h *APIHandler) GetHostElasticProcess(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { // }
hostID := ps.MustGetParameter("host_id") // if hostInfo.AgentID == "" {
hostInfo := &host.HostInfo{} // h.WriteJSON(w, util.MapStr{}, http.StatusOK)
hostInfo.ID = hostID // return
exists, err := orm.Get(hostInfo) // }
if err != nil { // sm := state.GetStateManager()
log.Error(err) // ag, err := sm.GetAgent(hostInfo.AgentID)
h.WriteError(w, err.Error(), http.StatusInternalServerError) // if err != nil {
return // log.Error(err)
} // h.WriteJSON(w, util.MapStr{}, http.StatusOK)
if !exists { // return
h.WriteError(w, fmt.Sprintf("host [%s] not found", hostID), http.StatusNotFound) // }
return // ctx,cancel := context.WithTimeout(context.Background(), time.Second * 10)
} // defer cancel()
if hostInfo.AgentID == "" { // esNodesInfo, err := sm.GetAgentClient().GetElasticsearchNodes(ctx, ag.GetEndpoint())
h.WriteJSON(w, util.MapStr{}, http.StatusOK) // if err != nil {
return // log.Error(err)
} // h.WriteError(w, err.Error(), http.StatusInternalServerError)
sm := state.GetStateManager() // return
ag, err := sm.GetAgent(hostInfo.AgentID) // }
if err != nil { // var processes []util.MapStr
log.Error(err) // for _, node := range esNodesInfo {
h.WriteJSON(w, util.MapStr{}, http.StatusOK) // processes = append(processes, util.MapStr{
return // "pid": node.ProcessInfo.PID,
} // "pid_status": node.ProcessInfo.Status,
ctx,cancel := context.WithTimeout(context.Background(), time.Second * 10) // "cluster_name": node.ClusterName,
defer cancel() // "cluster_uuid": node.ClusterUuid,
esNodesInfo, err := sm.GetAgentClient().GetElasticsearchNodes(ctx, ag.GetEndpoint()) // "cluster_id": node.ClusterID,
if err != nil { // "node_id": node.NodeUUID,
log.Error(err) // "node_name": node.NodeName,
h.WriteError(w, err.Error(), http.StatusInternalServerError) // "uptime_in_ms": time.Now().UnixMilli() - node.ProcessInfo.CreateTime,
return // })
} // }
var processes []util.MapStr // h.WriteJSON(w, util.MapStr{
for _, node := range esNodesInfo { // "elastic_processes": processes,
processes = append(processes, util.MapStr{ // }, http.StatusOK)
"pid": node.ProcessInfo.PID, //}
"pid_status": node.ProcessInfo.Status, //
"cluster_name": node.ClusterName, //func enrollHostFromAgent(agentID string) (*host.HostInfo, error){
"cluster_uuid": node.ClusterUuid, // sm := state.GetStateManager()
"cluster_id": node.ClusterID, // ag, err := sm.GetAgent(agentID)
"node_id": node.NodeUUID, // if err != nil {
"node_name": node.NodeName, // return nil, err
"uptime_in_ms": time.Now().UnixMilli() - node.ProcessInfo.CreateTime, // }
}) // if ag == nil {
} // return nil, fmt.Errorf("can not found agent [%s]", agentID)
h.WriteJSON(w, util.MapStr{ // }
"elastic_processes": processes, // agentClient := sm.GetAgentClient()
}, http.StatusOK) // hostInfo, err := agentClient.GetHostInfo(nil, ag.GetEndpoint())
} // if err != nil {
// return nil, err
func enrollHostFromAgent(agentID string) (*host.HostInfo, error){ // }
sm := state.GetStateManager() // hostInfo.AgentStatus = ag.Status
ag, err := sm.GetAgent(agentID) // return hostInfo, nil
if err != nil { //}
return nil, err
}
if ag == nil {
return nil, fmt.Errorf("can not found agent [%s]", agentID)
}
agentClient := sm.GetAgentClient()
hostInfo, err := agentClient.GetHostInfo(nil, ag.GetEndpoint())
if err != nil {
return nil, err
}
hostInfo.AgentStatus = ag.Status
return hostInfo, nil
}

55
modules/agent/api/init.go
View File

@ -4,35 +4,32 @@
package api package api
import (
"infini.sh/framework/core/api"
"infini.sh/framework/core/api/rbac/enum"
)
func Init() { func Init() {
handler := APIHandler{} //handler := APIHandler{}
api.HandleAPIMethod(api.POST, "/agent/instance", handler.createInstance) //api.HandleAPIMethod(api.POST, "/instance", handler.registerInstance) //new
api.HandleAPIMethod(api.GET, "/agent/instance/_search", handler.RequirePermission(handler.searchInstance, enum.PermissionAgentInstanceRead))
api.HandleAPIMethod(api.GET, "/agent/instance/:instance_id", handler.getInstance)
api.HandleAPIMethod(api.PUT, "/agent/instance/:instance_id", handler.updateInstance)
api.HandleAPIMethod(api.DELETE, "/agent/instance/:instance_id", handler.RequirePermission(handler.deleteInstance, enum.PermissionAgentInstanceWrite))
api.HandleAPIMethod(api.POST, "/agent/instance/_stats", handler.RequirePermission(handler.getInstanceStats, enum.PermissionAgentInstanceRead))
api.HandleAPIMethod(api.GET, "/agent/log/node/:node_id/files", handler.RequirePermission(handler.getLogFilesByNode, enum.PermissionAgentInstanceRead))
api.HandleAPIMethod(api.POST, "/agent/log/node/:node_id/_scroll", handler.RequirePermission(handler.getLogFileContent, enum.PermissionAgentInstanceRead))
api.HandleAPIMethod(api.GET, "/agent/instance/:instance_id/_nodes", handler.RequirePermission(handler.getESNodesInfo, enum.PermissionAgentInstanceRead))
api.HandleAPIMethod(api.POST, "/agent/instance/:instance_id/_nodes/_refresh", handler.RequirePermission(handler.refreshESNodesInfo, enum.PermissionAgentInstanceWrite))
api.HandleAPIMethod(api.POST, "/agent/instance/:instance_id/node/_auth", handler.RequirePermission(handler.authESNode, enum.PermissionAgentInstanceWrite))
api.HandleAPIMethod(api.DELETE, "/agent/instance/:instance_id/_nodes", handler.RequirePermission(handler.deleteESNode, enum.PermissionAgentInstanceWrite))
api.HandleAPIMethod(api.POST, "/agent/instance/:instance_id/node/_associate", handler.RequirePermission(handler.associateESNode, enum.PermissionAgentInstanceWrite))
api.HandleAPIMethod(api.POST, "/agent/instance/try_connect", handler.RequireLogin(handler.tryConnect))
api.HandleAPIMethod(api.POST, "/agent/auto_associate", handler.RequirePermission(handler.autoAssociateESNode, enum.PermissionAgentInstanceWrite))
api.HandleAPIMethod(api.POST, "/host/_enroll", handler.enrollHost) //api.HandleAPIMethod(api.POST, "/agent/instance", handler.registerInstance)
api.HandleAPIMethod(api.GET, "/host/:host_id/agent/info",handler.GetHostAgentInfo) //api.HandleAPIMethod(api.GET, "/agent/instance/_search", handler.RequirePermission(handler.searchInstance, enum.PermissionAgentInstanceRead))
api.HandleAPIMethod(api.GET, "/host/:host_id/processes",handler.GetHostElasticProcess) //api.HandleAPIMethod(api.GET, "/agent/instance/:instance_id", handler.getInstance)
api.HandleAPIMethod(api.DELETE, "/host/:host_id",handler.deleteHost) //api.HandleAPIMethod(api.PUT, "/agent/instance/:instance_id", handler.updateInstance)
//api.HandleAPIMethod(api.DELETE, "/agent/instance/:instance_id", handler.RequirePermission(handler.deleteInstance, enum.PermissionAgentInstanceWrite))
//api.HandleAPIMethod(api.POST, "/agent/instance/_stats", handler.RequirePermission(handler.getInstanceStats, enum.PermissionAgentInstanceRead))
api.HandleAPIMethod(api.POST, "/agent/install_command", handler.RequireLogin(handler.generateInstallCommand)) //api.HandleAPIMethod(api.GET, "/agent/log/node/:node_id/files", handler.RequirePermission(handler.getLogFilesByNode, enum.PermissionAgentInstanceRead))
api.HandleAPIMethod(api.GET, "/agent/install.sh", handler.getInstallScript) //api.HandleAPIMethod(api.POST, "/agent/log/node/:node_id/_scroll", handler.RequirePermission(handler.getLogFileContent, enum.PermissionAgentInstanceRead))
//api.HandleAPIMethod(api.GET, "/agent/instance/:instance_id/_nodes", handler.RequirePermission(handler.getESNodesInfo, enum.PermissionAgentInstanceRead))
//api.HandleAPIMethod(api.POST, "/agent/instance/:instance_id/_nodes/_refresh", handler.RequirePermission(handler.refreshESNodesInfo, enum.PermissionAgentInstanceWrite))
//api.HandleAPIMethod(api.POST, "/agent/instance/:instance_id/node/_auth", handler.RequirePermission(handler.authESNode, enum.PermissionAgentInstanceWrite))
//api.HandleAPIMethod(api.DELETE, "/agent/instance/:instance_id/_nodes", handler.RequirePermission(handler.deleteESNode, enum.PermissionAgentInstanceWrite))
//api.HandleAPIMethod(api.POST, "/agent/instance/:instance_id/node/_associate", handler.RequirePermission(handler.associateESNode, enum.PermissionAgentInstanceWrite))
//api.HandleAPIMethod(api.POST, "/agent/instance/try_connect", handler.RequireLogin(handler.tryConnect))
//api.HandleAPIMethod(api.POST, "/agent/auto_associate", handler.RequirePermission(handler.autoAssociateESNode, enum.PermissionAgentInstanceWrite))
//
//api.HandleAPIMethod(api.POST, "/host/_enroll", handler.enrollHost)
//api.HandleAPIMethod(api.GET, "/host/:host_id/agent/info",handler.GetHostAgentInfo)
//api.HandleAPIMethod(api.GET, "/host/:host_id/processes",handler.GetHostElasticProcess)
//api.HandleAPIMethod(api.DELETE, "/host/:host_id",handler.deleteHost)
//
//
//api.HandleAPIMethod(api.POST, "/agent/install_command", handler.RequireLogin(handler.generateInstallCommand))
//api.HandleAPIMethod(api.GET, "/agent/install.sh", handler.getInstallScript)
} }

1143
modules/agent/api/instance.go
View File

File diff suppressed because it is too large Load Diff

20
modules/agent/api/log.go
View File

@ -9,8 +9,8 @@ import (
log "github.com/cihub/seelog" log "github.com/cihub/seelog"
"infini.sh/console/modules/agent/client" "infini.sh/console/modules/agent/client"
"infini.sh/console/modules/agent/state" "infini.sh/console/modules/agent/state"
"infini.sh/framework/core/agent"
httprouter "infini.sh/framework/core/api/router" httprouter "infini.sh/framework/core/api/router"
"infini.sh/framework/core/model"
"infini.sh/framework/core/orm" "infini.sh/framework/core/orm"
"infini.sh/framework/core/util" "infini.sh/framework/core/util"
"net/http" "net/http"
@ -80,7 +80,7 @@ func (h *APIHandler) getLogFileContent(w http.ResponseWriter, req *http.Request,
h.WriteJSON(w, res, http.StatusOK) h.WriteJSON(w, res, http.StatusOK)
} }
func getAgentByNodeID(nodeID string) (*agent.Instance, *agent.ESNodeInfo, error){ func getAgentByNodeID(nodeID string) (*model.Instance, *model.ESNodeInfo, error){
queryDsl := util.MapStr{ queryDsl := util.MapStr{
"size":1, "size":1,
"query": util.MapStr{ "query": util.MapStr{
@ -101,24 +101,24 @@ func getAgentByNodeID(nodeID string) (*agent.Instance, *agent.ESNodeInfo, error)
q := &orm.Query{ q := &orm.Query{
RawQuery: util.MustToJSONBytes(queryDsl), RawQuery: util.MustToJSONBytes(queryDsl),
} }
err, result := orm.Search(agent.ESNodeInfo{}, q) err, result := orm.Search(model.ESNodeInfo{}, q)
if err != nil { if err != nil {
return nil,nil, err return nil,nil, err
} }
if len(result.Result) > 0 { if len(result.Result) > 0 {
buf := util.MustToJSONBytes(result.Result[0]) buf := util.MustToJSONBytes(result.Result[0])
node := &agent.ESNodeInfo{} v := &model.ESNodeInfo{}
err = util.FromJSONBytes(buf, node) err = util.FromJSONBytes(buf, v)
inst := &agent.Instance{} inst := &model.Instance{}
inst.ID = node.AgentID inst.ID = v.AgentID
_, err = orm.Get(inst) _, err = orm.Get(inst)
if err != nil { if err != nil {
return nil, node, err return nil, v, err
} }
if inst.Name == "" { if inst.Name == "" {
return nil, node, nil return nil, v, nil
} }
return inst, node, nil return inst, v, nil
} }
return nil, nil, nil return nil, nil, nil
} }

155
modules/agent/api/setup.go
View File

@ -1,155 +0,0 @@
/* Copyright © INFINI Ltd. All rights reserved.
* Web: https://infinilabs.com
* Email: hello#infini.ltd */
package api
import (
"fmt"
log "github.com/cihub/seelog"
"github.com/valyala/fasttemplate"
"infini.sh/console/modules/agent/common"
"infini.sh/framework/core/api/rbac"
httprouter "infini.sh/framework/core/api/router"
"infini.sh/framework/core/global"
"infini.sh/framework/core/util"
"os"
"net/http"
"path"
"strings"
"sync"
"time"
)
var tokens = sync.Map{}
type Token struct {
CreatedAt time.Time
UserID string
}
const ExpiredIn = time.Millisecond * 1000 * 60 * 60
func (h *APIHandler) generateInstallCommand(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
claims, ok := req.Context().Value("user").(*rbac.UserClaims)
if !ok {
h.WriteError(w, "user not found", http.StatusInternalServerError)
return
}
agCfg := common.GetAgentConfig()
if agCfg == nil || agCfg.Setup == nil {
h.WriteError(w, "agent setup config was not found, please configure in the configuration file first", http.StatusInternalServerError)
return
}
var (
t *Token
tokenStr string
)
tokens.Range(func(key, value any) bool {
if v, ok := value.(*Token); ok && claims.UserId == v.UserID {
t = v
tokenStr = key.(string)
return false
}
return true
})
if t == nil {
tokenStr = util.GetUUID()
t = &Token{
CreatedAt: time.Now(),
UserID: claims.UserId,
}
}else{
if t.CreatedAt.Add(ExpiredIn).Before(time.Now()){
tokens.Delete(tokenStr)
tokenStr = util.GetUUID()
t = &Token{
CreatedAt: time.Now(),
UserID: claims.UserId,
}
}else{
t.CreatedAt = time.Now()
}
}
tokens.Store(tokenStr, t)
consoleEndpoint := agCfg.Setup.ConsoleEndpoint
if consoleEndpoint == "" {
consoleEndpoint = getDefaultConsoleEndpoint(req)
}
h.WriteJSON(w, util.MapStr{
"script": fmt.Sprintf(`curl -sSL %s/agent/install.sh?token=%s |sudo bash -s -- -u %s -v %s -t /opt/agent`, consoleEndpoint, tokenStr, agCfg.Setup.DownloadURL, agCfg.Setup.Version),
//"script": fmt.Sprintf(`sudo BASE_URL="%s" AGENT_VER="%s" INSTALL_PATH="/opt" bash -c "$(curl -L '%s/agent/install.sh?token=%s')"`, agCfg.Setup.DownloadURL, agCfg.Setup.Version, consoleEndpoint, tokenStr),
"token": tokenStr,
"expired_at": t.CreatedAt.Add(ExpiredIn),
}, http.StatusOK)
}
func getDefaultConsoleEndpoint(req *http.Request) string{
scheme := "http"
if req.TLS != nil {
scheme = "https"
}
return fmt.Sprintf("%s://%s", scheme, req.Host)
}
func (h *APIHandler) getInstallScript(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
tokenStr := h.GetParameter(req, "token")
if strings.TrimSpace(tokenStr) == "" {
h.WriteError(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
return
}
if v, ok := tokens.Load(tokenStr); !ok {
h.WriteError(w, "token is invalid", http.StatusUnauthorized)
return
}else{
if t, ok := v.(*Token); !ok || t.CreatedAt.Add(ExpiredIn).Before(time.Now()) {
tokens.Delete(tokenStr)
h.WriteError(w, "token was expired", http.StatusUnauthorized)
return
}
}
agCfg := common.GetAgentConfig()
caCert, clientCertPEM, clientKeyPEM, err := common.GenerateServerCert(agCfg.Setup.CACertFile, agCfg.Setup.CAKeyFile)
if err != nil {
log.Error(err)
h.WriteError(w, err.Error(), http.StatusInternalServerError)
return
}
scriptTplPath := path.Join(global.Env().GetConfigDir(), "install_agent.tpl")
buf, err := os.ReadFile(scriptTplPath)
if err != nil {
log.Error(err)
h.WriteError(w, err.Error(), http.StatusInternalServerError)
return
}
tpl := fasttemplate.New(string(buf), "{{", "}}")
downloadURL := agCfg.Setup.DownloadURL
if downloadURL == "" {
downloadURL = "https://release.infinilabs.com/agent/stable/"
}
port := agCfg.Setup.Port
if port == "" {
port = "8080"
}
consoleEndpoint := agCfg.Setup.ConsoleEndpoint
if consoleEndpoint == "" {
consoleEndpoint = getDefaultConsoleEndpoint(req)
}
_, err = tpl.Execute(w, map[string]interface{}{
"base_url": agCfg.Setup.DownloadURL,
"agent_version": agCfg.Setup.Version,
"console_endpoint": consoleEndpoint,
"client_crt": clientCertPEM,
"client_key": clientKeyPEM,
"ca_crt": caCert,
"port": port,
"token": tokenStr,
})
if err != nil {
log.Error(err)
}
}

79
modules/agent/client/client.go
View File

@ -8,7 +8,7 @@ import (
"context" "context"
"fmt" "fmt"
"infini.sh/console/modules/agent/common" "infini.sh/console/modules/agent/common"
"infini.sh/framework/core/agent" "infini.sh/framework/core/model"
"infini.sh/framework/core/elastic" "infini.sh/framework/core/elastic"
"infini.sh/framework/core/host" "infini.sh/framework/core/host"
"infini.sh/framework/core/util" "infini.sh/framework/core/util"
@ -32,10 +32,10 @@ type ClientAPI interface {
GetElasticProcess(ctx context.Context, agentBaseURL string, agentID string)(interface{}, error) GetElasticProcess(ctx context.Context, agentBaseURL string, agentID string)(interface{}, error)
GetElasticLogFiles(ctx context.Context, agentBaseURL string, logsPath string)(interface{}, error) GetElasticLogFiles(ctx context.Context, agentBaseURL string, logsPath string)(interface{}, error)
GetElasticLogFileContent(ctx context.Context, agentBaseURL string, body interface{})(interface{}, error) GetElasticLogFileContent(ctx context.Context, agentBaseURL string, body interface{})(interface{}, error)
GetInstanceBasicInfo(ctx context.Context, agentBaseURL string) (*agent.Instance, error) GetInstanceBasicInfo(ctx context.Context, agentBaseURL string) (*model.Instance, error)
RegisterElasticsearch(ctx context.Context, agentBaseURL string, cfgs []elastic.ElasticsearchConfig) error RegisterElasticsearch(ctx context.Context, agentBaseURL string, cfgs []elastic.ElasticsearchConfig) error
GetElasticsearchNodes(ctx context.Context, agentBaseURL string) ([]agent.ESNodeInfo, error) GetElasticsearchNodes(ctx context.Context, agentBaseURL string) ([]model.ESNodeInfo, error)
AuthESNode(ctx context.Context, agentBaseURL string, cfg elastic.ElasticsearchConfig) (*agent.ESNodeInfo, error) AuthESNode(ctx context.Context, agentBaseURL string, cfg elastic.ElasticsearchConfig) (*model.ESNodeInfo, error)
CreatePipeline(ctx context.Context, agentBaseURL string, body []byte) error CreatePipeline(ctx context.Context, agentBaseURL string, body []byte) error
DeletePipeline(ctx context.Context, agentBaseURL, pipelineID string) error DeletePipeline(ctx context.Context, agentBaseURL, pipelineID string) error
SetKeystoreValue(ctx context.Context, agentBaseURL string, key, value string) error SetKeystoreValue(ctx context.Context, agentBaseURL string, key, value string) error
@ -49,28 +49,30 @@ type Client struct {
} }
func (client *Client) GetHostInfo(ctx context.Context, agentBaseURL string) (*host.HostInfo, error) { //func (client *Client) GetHostInfo(ctx context.Context, agentBaseURL string) (*host.HostInfo, error) {
req := &util.Request{ // req := &util.Request{
Method: http.MethodGet, // Method: http.MethodGet,
Url: fmt.Sprintf("%s/agent/host/_basic", agentBaseURL), // Url: fmt.Sprintf("%s/agent/host/_basic", agentBaseURL),
Context: ctx, // Context: ctx,
} // }
resBody := struct { // resBody := struct {
Success bool `json:"success"` // Success bool `json:"success"`
Error string `json:"error"` // Error string `json:"error"`
HostInfo *host.HostInfo `json:"result"` // HostInfo *host.HostInfo `json:"result"`
}{} // }{}
err := client.DoRequest(req, &resBody) // err := client.DoRequest(req, &resBody)
if err != nil { // if err != nil {
return nil, err // return nil, err
} // }
if resBody.Success != true { // if resBody.Success != true {
return nil, fmt.Errorf("enroll error from client: %v", resBody.Error) // return nil, fmt.Errorf("enroll error from client: %v", resBody.Error)
} // }
return resBody.HostInfo, nil // return resBody.HostInfo, nil
} //}
//TODO
func (client *Client) GetElasticProcess(ctx context.Context, agentBaseURL string, agentID string)(interface{}, error) { func (client *Client) GetElasticProcess(ctx context.Context, agentBaseURL string, agentID string)(interface{}, error) {
panic("implement me")
req := &util.Request{ req := &util.Request{
Method: http.MethodGet, Method: http.MethodGet,
Url: fmt.Sprintf("%s/elasticsearch/%s/process/_elastic", agentBaseURL, agentID), Url: fmt.Sprintf("%s/elasticsearch/%s/process/_elastic", agentBaseURL, agentID),
@ -88,6 +90,9 @@ func (client *Client) GetElasticProcess(ctx context.Context, agentBaseURL string
} }
func (client *Client) GetElasticLogFiles(ctx context.Context, agentBaseURL string, logsPath string)(interface{}, error) { func (client *Client) GetElasticLogFiles(ctx context.Context, agentBaseURL string, logsPath string)(interface{}, error) {
panic("implement me")
reqBody := util.MustToJSONBytes(util.MapStr{ reqBody := util.MustToJSONBytes(util.MapStr{
"logs_path": logsPath, "logs_path": logsPath,
}) })
@ -133,13 +138,13 @@ func (client *Client) GetElasticLogFileContent(ctx context.Context, agentBaseURL
} , nil } , nil
} }
func (client *Client) GetInstanceBasicInfo(ctx context.Context, agentBaseURL string) (*agent.Instance, error){ func (client *Client) GetInstanceBasicInfo(ctx context.Context, agentBaseURL string) (*model.Instance, error){
req := &util.Request{ req := &util.Request{
Method: http.MethodGet, Method: http.MethodGet,
Url: fmt.Sprintf("%s/agent/_info", agentBaseURL ), Url: fmt.Sprintf("%s/_info", agentBaseURL ),
Context: ctx, Context: ctx,
} }
resBody := &agent.Instance{} resBody := &model.Instance{}
err := client.DoRequest(req, &resBody) err := client.DoRequest(req, &resBody)
return resBody, err return resBody, err
} }
@ -166,13 +171,13 @@ func (client *Client) RegisterElasticsearch(ctx context.Context, agentBaseURL st
return nil return nil
} }
func (client *Client) GetElasticsearchNodes(ctx context.Context, agentBaseURL string) ([]agent.ESNodeInfo, error) { func (client *Client) GetElasticsearchNodes(ctx context.Context, agentBaseURL string) ([]model.ESNodeInfo, error) {
req := &util.Request{ req := &util.Request{
Method: http.MethodGet, Method: http.MethodGet,
Url: fmt.Sprintf("%s/elasticsearch/_nodes", agentBaseURL ), Url: fmt.Sprintf("%s/elasticsearch/nodes/_discovery", agentBaseURL ),
Context: ctx, Context: ctx,
} }
resBody := []agent.ESNodeInfo{} resBody := []model.ESNodeInfo{}
err := client.DoRequest(req, &resBody) err := client.DoRequest(req, &resBody)
if err != nil { if err != nil {
return nil, err return nil, err
@ -181,7 +186,7 @@ func (client *Client) GetElasticsearchNodes(ctx context.Context, agentBaseURL st
return resBody, nil return resBody, nil
} }
func (client *Client) AuthESNode(ctx context.Context, agentBaseURL string, cfg elastic.ElasticsearchConfig) (*agent.ESNodeInfo, error) { func (client *Client) AuthESNode(ctx context.Context, agentBaseURL string, cfg elastic.ElasticsearchConfig) (*model.ESNodeInfo, error) {
reqBody, err := util.ToJSONBytes(cfg) reqBody, err := util.ToJSONBytes(cfg)
if err != nil { if err != nil {
return nil, err return nil, err
@ -192,7 +197,7 @@ func (client *Client) AuthESNode(ctx context.Context, agentBaseURL string, cfg e
Context: ctx, Context: ctx,
Body: reqBody, Body: reqBody,
} }
resBody := &agent.ESNodeInfo{} resBody := &model.ESNodeInfo{}
err = client.DoRequest(req, resBody) err = client.DoRequest(req, resBody)
if err != nil { if err != nil {
return nil, err return nil, err
@ -227,22 +232,22 @@ func (client *Client) SetKeystoreValue(ctx context.Context, agentBaseURL string,
} }
req := &util.Request{ req := &util.Request{
Method: http.MethodPost, Method: http.MethodPost,
Url: fmt.Sprintf("%s/_framework/keystore", agentBaseURL), Url: fmt.Sprintf("%s/keystore", agentBaseURL),
Context: ctx, Context: ctx,
Body: util.MustToJSONBytes(body), Body: util.MustToJSONBytes(body),
} }
return client.DoRequest(req, nil) return client.DoRequest(req, nil)
} }
func (client *Client) SaveDynamicConfig(ctx context.Context, agentBaseURL string, name, content string) error{ func (client *Client) SaveDynamicConfig(ctx context.Context, agentBaseURL string, filename, content string) error{
body := util.MapStr{ body := util.MapStr{
"configs": util.MapStr{ "configs": util.MapStr{
name: content, filename: content,
}, },
} }
req := &util.Request{ req := &util.Request{
Method: http.MethodPost, Method: http.MethodPost,
Url: fmt.Sprintf("%s/agent/config", agentBaseURL), Url: fmt.Sprintf("%s/config/_update", agentBaseURL),
Context: ctx, Context: ctx,
Body: util.MustToJSONBytes(body), Body: util.MustToJSONBytes(body),
} }
@ -260,7 +265,7 @@ func (client *Client) SaveIngestConfig(ctx context.Context, agentBaseURL string)
return fmt.Errorf("set keystore value to agent error: %w", err) return fmt.Errorf("set keystore value to agent error: %w", err)
} }
} }
err = client.SaveDynamicConfig(context.Background(), agentBaseURL, "ingest", ingestCfg ) err = client.SaveDynamicConfig(context.Background(), agentBaseURL, "ingest_variables.yml", ingestCfg )
if err != nil { if err != nil {
return fmt.Errorf("save dynamic config to agent error: %w", err) return fmt.Errorf("save dynamic config to agent error: %w", err)
} }

138
modules/agent/common/helper.go
View File

@ -6,19 +6,19 @@ package common
import ( import (
"fmt" "fmt"
"infini.sh/console/modules/agent/model" log "github.com/cihub/seelog"
"infini.sh/framework/core/agent" model2 "infini.sh/console/modules/agent/model"
"infini.sh/framework/core/credential" "infini.sh/framework/core/credential"
"infini.sh/framework/core/elastic" "infini.sh/framework/core/elastic"
"infini.sh/framework/core/event" "infini.sh/framework/core/event"
"infini.sh/framework/core/global" "infini.sh/framework/core/global"
"infini.sh/framework/core/model"
"infini.sh/framework/core/orm" "infini.sh/framework/core/orm"
"infini.sh/framework/core/util" "infini.sh/framework/core/util"
log "src/github.com/cihub/seelog"
"strings" "strings"
) )
func ParseAgentSettings(settings []agent.Setting)(*model.ParseAgentSettingsResult, error){ func ParseAgentSettings(settings []model.Setting)(*model2.ParseAgentSettingsResult, error){
var clusterCfgs []elastic.ElasticsearchConfig var clusterCfgs []elastic.ElasticsearchConfig
var ( var (
pipelines []util.MapStr pipelines []util.MapStr
@ -58,7 +58,7 @@ func ParseAgentSettings(settings []agent.Setting)(*model.ParseAgentSettingsResul
if err != nil { if err != nil {
return nil, err return nil, err
} }
taskSetting := model.TaskSetting{} taskSetting := model2.TaskSetting{}
err = util.FromJSONBytes(vBytes, &taskSetting) err = util.FromJSONBytes(vBytes, &taskSetting)
if err != nil { if err != nil {
return nil, err return nil, err
@ -70,7 +70,7 @@ func ParseAgentSettings(settings []agent.Setting)(*model.ParseAgentSettingsResul
pipelines = append(pipelines, partPipelines...) pipelines = append(pipelines, partPipelines...)
toDeletePipelineNames = append(toDeletePipelineNames, partDeletePipelineNames...) toDeletePipelineNames = append(toDeletePipelineNames, partDeletePipelineNames...)
} }
return &model.ParseAgentSettingsResult{ return &model2.ParseAgentSettingsResult{
ClusterConfigs: clusterCfgs, ClusterConfigs: clusterCfgs,
Pipelines: pipelines, Pipelines: pipelines,
ToDeletePipelineNames: toDeletePipelineNames, ToDeletePipelineNames: toDeletePipelineNames,
@ -80,7 +80,7 @@ func ParseAgentSettings(settings []agent.Setting)(*model.ParseAgentSettingsResul
// GetAgentSettings query agent setting by agent id and updated timestamp, // GetAgentSettings query agent setting by agent id and updated timestamp,
// if there has any setting was updated, then return setting list includes settings not changed, // if there has any setting was updated, then return setting list includes settings not changed,
// otherwise return empty setting list // otherwise return empty setting list
func GetAgentSettings(agentID string, timestamp int64) ([]agent.Setting, error) { func GetAgentSettings(agentID string, timestamp int64) ([]model.Setting, error) {
query := util.MapStr{ query := util.MapStr{
"bool": util.MapStr{ "bool": util.MapStr{
"must": []util.MapStr{ "must": []util.MapStr{
@ -116,13 +116,13 @@ func GetAgentSettings(agentID string, timestamp int64) ([]agent.Setting, error)
}, },
} }
queryDsl := util.MapStr{ queryDsl := util.MapStr{
"size": 500, "size": 1000,
"query": query, "query": query,
} }
q := orm.Query{ q := orm.Query{
RawQuery: util.MustToJSONBytes(queryDsl), RawQuery: util.MustToJSONBytes(queryDsl),
} }
err, result := orm.Search(agent.Setting{}, &q) err, result := orm.Search(model.Setting{}, &q)
if err != nil { if err != nil {
return nil, fmt.Errorf("search settings error: %w", err) return nil, fmt.Errorf("search settings error: %w", err)
} }
@ -130,11 +130,11 @@ func GetAgentSettings(agentID string, timestamp int64) ([]agent.Setting, error)
return nil, nil return nil, nil
} }
var ( var (
settings []agent.Setting settings []model.Setting
hasUpdated bool hasUpdated bool
) )
for _, row := range result.Result { for _, row := range result.Result {
setting := agent.Setting{} setting := model.Setting{}
buf, err := util.ToJSONBytes(row) buf, err := util.ToJSONBytes(row)
if err != nil { if err != nil {
return nil, err return nil, err
@ -158,7 +158,7 @@ func getClusterConfigReferenceName(clusterID, nodeUUID string) string {
return fmt.Sprintf("%s_%s", clusterID, nodeUUID) return fmt.Sprintf("%s_%s", clusterID, nodeUUID)
} }
func TransformSettingsToConfig(setting *model.TaskSetting, clusterID, nodeUUID string) ([]util.MapStr, []string, error) { func TransformSettingsToConfig(setting *model2.TaskSetting, clusterID, nodeUUID string) ([]util.MapStr, []string, error) {
if setting == nil { if setting == nil {
return nil, nil, fmt.Errorf("empty setting") return nil, nil, fmt.Errorf("empty setting")
} }
@ -279,6 +279,7 @@ func newClusterMetricPipeline(processorName string, clusterID string, nodeUUID s
"name": getMetricPipelineName(clusterID, processorName), "name": getMetricPipelineName(clusterID, processorName),
"auto_start": true, "auto_start": true,
"keep_running": true, "keep_running": true,
"singleton": true,
"retry_delay_in_ms": 10000, "retry_delay_in_ms": 10000,
"processor": []util.MapStr{cfg}, "processor": []util.MapStr{cfg},
} }
@ -290,22 +291,22 @@ func getMetricPipelineName(clusterID, processorName string) string{
} }
func LoadAgentsFromES(clusterID string) ([]agent.Instance, error) { func LoadAgentsFromES(clusterID string) ([]model.Instance, error) {
q := orm.Query{ q := orm.Query{
Size: 1000, Size: 1000,
} }
if clusterID != "" { if clusterID != "" {
q.Conds = orm.And(orm.Eq("id", clusterID)) q.Conds = orm.And(orm.Eq("id", clusterID))
} }
err, result := orm.Search(agent.Instance{}, &q) err, result := orm.Search(model.Instance{}, &q)
if err != nil { if err != nil {
return nil, fmt.Errorf("query agent error: %w", err) return nil, fmt.Errorf("query agent error: %w", err)
} }
if len(result.Result) > 0 { if len(result.Result) > 0 {
var agents = make([]agent.Instance, 0, len(result.Result)) var agents = make([]model.Instance, 0, len(result.Result))
for _, row := range result.Result { for _, row := range result.Result {
ag := agent.Instance{} ag := model.Instance{}
bytes := util.MustToJSONBytes(row) bytes := util.MustToJSONBytes(row)
err = util.FromJSONBytes(bytes, &ag) err = util.FromJSONBytes(bytes, &ag)
if err != nil { if err != nil {
@ -417,7 +418,7 @@ func GetAgentIngestConfig() (string, *elastic.BasicAuth, error) {
} }
if emptyIngestClusterEndpoint { if emptyIngestClusterEndpoint {
cfg := elastic.GetConfig(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) cfg := elastic.GetConfig(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
endpoint = cfg.Endpoint endpoint = cfg.GetAnyEndpoint()
} }
var ( var (
@ -441,101 +442,14 @@ func GetAgentIngestConfig() (string, *elastic.BasicAuth, error) {
cfg := elastic.GetConfig(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) cfg := elastic.GetConfig(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
basicAuth = *cfg.BasicAuth basicAuth = *cfg.BasicAuth
} }
tpl := `elasticsearch: tpl := `configs.template:
- name: default - name: "ingest"
enabled: true path: ./config/ingest_config.tpl
endpoint: %s variable:
discovery: INGEST_CLUSTER_ID: "default_ingest_cluster"
enabled: true INGEST_CLUSTER_ENDPOINT: ["%s"]
basic_auth: INGEST_CLUSTER_USERNAME: "%s"
username: %s `
password: $[[keystore.ingest_cluster_password]]
metrics:
enabled: true
queue: metrics
network:
enabled: true
summary: true
details: true
memory:
metrics:
- swap
- memory
disk:
metrics:
- iops
- usage
cpu:
metrics:
- idle
- system
- user
- iowait
- load
instance:
enabled: true
pipeline:
- name: logs_indexing_merge
auto_start: true
keep_running: true
processor:
- indexing_merge:
index_name: ".infini_logs"
elasticsearch: "default"
input_queue: "logs"
idle_timeout_in_seconds: 10
output_queue:
name: "logs_requests"
label:
tag: "logs"
worker_size: 1
bulk_size_in_mb: 10
- name: ingest_logs
auto_start: true
keep_running: true
processor:
- bulk_indexing:
bulk:
compress: true
batch_size_in_mb: 5
batch_size_in_docs: 5000
consumer:
fetch_max_messages: 100
queues:
type: indexing_merge
tag: "logs"
when:
cluster_available: ["default"]
- name: metrics_indexing_merge
auto_start: true
keep_running: true
processor:
- indexing_merge:
elasticsearch: "default"
index_name: ".infini_metrics"
input_queue: "metrics"
output_queue:
name: "metrics_requests"
label:
tag: "metrics"
worker_size: 1
bulk_size_in_mb: 5
- name: ingest_metrics
auto_start: true
keep_running: true
processor:
- bulk_indexing:
bulk:
compress: true
batch_size_in_mb: 5
batch_size_in_docs: 5000
consumer:
fetch_max_messages: 100
queues:
type: indexing_merge
tag: "metrics"
when:
cluster_available: ["default"]`
tpl = fmt.Sprintf(tpl, endpoint, basicAuth.Username) tpl = fmt.Sprintf(tpl, endpoint, basicAuth.Username)
return tpl, &basicAuth, nil return tpl, &basicAuth, nil
} }

20
modules/agent/model/config.go
View File

@ -5,20 +5,20 @@
package model package model
type AgentConfig struct { type AgentConfig struct {
Enabled bool `config:"enabled"` Enabled bool `config:"enabled"`
StateManager struct{ StateManager struct {
Enabled bool `config:"enabled"` Enabled bool `config:"enabled"`
} `config:"state_manager"` } `config:"state_manager"`
Setup *SetupConfig `config:"setup"` Setup *SetupConfig `config:"setup"`
} }
type SetupConfig struct { type SetupConfig struct {
DownloadURL string `config:"download_url"` DownloadURL string `config:"download_url"`
Version string `config:"version"` Version string `config:"version"`
CACertFile string `config:"ca_cert"` CACertFile string `config:"ca_cert"`
CAKeyFile string `config:"ca_key"` CAKeyFile string `config:"ca_key"`
ConsoleEndpoint string `config:"console_endpoint"` ConsoleEndpoint string `config:"console_endpoint"`
IngestClusterEndpoint interface{} `config:"ingest_cluster_endpoint"` IngestClusterEndpoint interface{} `config:"ingest_cluster_endpoint"`
IngestClusterCredentialID string `config:"ingest_cluster_credential_id"` IngestClusterCredentialID string `config:"ingest_cluster_credential_id"`
Port string `config:"port"` Port string `config:"port"`
} }

49
modules/agent/state/state.go
View File

@ -12,8 +12,8 @@ import (
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
"infini.sh/console/modules/agent/client" "infini.sh/console/modules/agent/client"
"infini.sh/console/modules/agent/common" "infini.sh/console/modules/agent/common"
"infini.sh/console/modules/agent/model" model2 "infini.sh/console/modules/agent/model"
"infini.sh/framework/core/agent" "infini.sh/framework/core/model"
"infini.sh/framework/core/host" "infini.sh/framework/core/host"
"infini.sh/framework/core/kv" "infini.sh/framework/core/kv"
"infini.sh/framework/core/orm" "infini.sh/framework/core/orm"
@ -44,9 +44,9 @@ func IsEnabled() bool {
} }
type IStateManager interface { type IStateManager interface {
GetAgent(ID string) (*agent.Instance, error) GetAgent(ID string) (*model.Instance, error)
UpdateAgent(inst *agent.Instance, syncToES bool) (*agent.Instance, error) UpdateAgent(inst *model.Instance, syncToES bool) (*model.Instance, error)
GetTaskAgent(clusterID string) (*agent.Instance, error) GetTaskAgent(clusterID string) (*model.Instance, error)
DeleteAgent(agentID string) error DeleteAgent(agentID string) error
LoopState() LoopState()
Stop() Stop()
@ -87,7 +87,7 @@ func (sm *StateManager) checkAgentStatus() {
for agentID := range onlineAgentIDs { for agentID := range onlineAgentIDs {
if _, ok := sm.agentIds[agentID]; !ok { if _, ok := sm.agentIds[agentID]; !ok {
log.Infof("status of agent [%s] changed to online", agentID) log.Infof("status of agent [%s] changed to online", agentID)
sm.agentIds[agentID] = model.StatusOnline sm.agentIds[agentID] = model2.StatusOnline
} }
} }
sm.agentMutex.Unlock() sm.agentMutex.Unlock()
@ -104,25 +104,25 @@ func (sm *StateManager) checkAgentStatus() {
sm.syncSettings(agentID) sm.syncSettings(agentID)
sm.syncIngestSettings(agentID) sm.syncIngestSettings(agentID)
if _, ok := onlineAgentIDs[agentID]; ok { if _, ok := onlineAgentIDs[agentID]; ok {
host.UpdateHostAgentStatus(agentID, model.StatusOnline) host.UpdateHostAgentStatus(agentID, model2.StatusOnline)
if status == model.StatusOnline { if status == model2.StatusOnline {
return return
} }
// status change to online // status change to online
sm.agentMutex.Lock() sm.agentMutex.Lock()
sm.agentIds[agentID] = model.StatusOnline sm.agentIds[agentID] = model2.StatusOnline
sm.agentMutex.Unlock() sm.agentMutex.Unlock()
log.Infof("status of agent [%s] changed to online", agentID) log.Infof("status of agent [%s] changed to online", agentID)
return return
}else{ }else{
// already offline // already offline
if status == model.StatusOffline { if status == model2.StatusOffline {
return return
} }
} }
// status change to offline // status change to offline
sm.agentMutex.Lock() sm.agentMutex.Lock()
sm.agentIds[agentID] = model.StatusOffline sm.agentIds[agentID] = model2.StatusOffline
sm.agentMutex.Unlock() sm.agentMutex.Unlock()
ag, err := sm.GetAgent(agentID) ag, err := sm.GetAgent(agentID)
if err != nil { if err != nil {
@ -131,7 +131,7 @@ func (sm *StateManager) checkAgentStatus() {
} }
return return
} }
ag.Status = model.StatusOffline ag.Status = model2.StatusOffline
log.Infof("agent [%s] is offline", ag.Endpoint) log.Infof("agent [%s] is offline", ag.Endpoint)
_, err = sm.UpdateAgent(ag, true) _, err = sm.UpdateAgent(ag, true)
if err != nil { if err != nil {
@ -139,13 +139,13 @@ func (sm *StateManager) checkAgentStatus() {
return return
} }
//update host agent status //update host agent status
host.UpdateHostAgentStatus(ag.ID, model.StatusOffline) host.UpdateHostAgentStatus(ag.ID, model2.StatusOffline)
}(agentID) }(agentID)
} }
} }
func (sm *StateManager) getLastSyncSettingsTimestamp(agentID string) int64{ func (sm *StateManager) getLastSyncSettingsTimestamp(agentID string) int64{
vbytes, err := kv.GetValue(model.KVSyncDynamicTaskSettings, []byte(agentID)) vbytes, err := kv.GetValue(model2.KVSyncDynamicTaskSettings, []byte(agentID))
if err != nil { if err != nil {
log.Error(err) log.Error(err)
} }
@ -191,7 +191,7 @@ func (sm *StateManager) syncSettings(agentID string) {
clusterCfg := util.MapStr{ clusterCfg := util.MapStr{
"name": cfg.ID, "name": cfg.ID,
"enabled": true, "enabled": true,
"endpoint": cfg.Endpoint, "endpoint": cfg.GetAnyEndpoint(),
} }
if cfg.BasicAuth != nil && cfg.BasicAuth.Password != ""{ if cfg.BasicAuth != nil && cfg.BasicAuth.Password != ""{
cid := cfg.ID cid := cfg.ID
@ -223,16 +223,17 @@ func (sm *StateManager) syncSettings(agentID string) {
log.Error("serialize config to yaml error: ", err) log.Error("serialize config to yaml error: ", err)
return return
} }
err = agClient.SaveDynamicConfig(context.Background(), ag.GetEndpoint(), "dynamic_task", string(cfgBytes)) //TODO
err = agClient.SaveDynamicConfig(context.Background(), ag.GetEndpoint(), "dynamic_task.yml", string(cfgBytes))
newTimestampStr := strconv.FormatInt(newTimestamp, 10) newTimestampStr := strconv.FormatInt(newTimestamp, 10)
err = kv.AddValue(model.KVSyncDynamicTaskSettings, []byte(agentID), []byte(newTimestampStr)) err = kv.AddValue(model2.KVSyncDynamicTaskSettings, []byte(agentID), []byte(newTimestampStr))
if err != nil { if err != nil {
log.Error(err) log.Error(err)
} }
} }
func (sm *StateManager) syncIngestSettings(agentID string) { func (sm *StateManager) syncIngestSettings(agentID string) {
v, err := kv.GetValue(model.KVAgentIngestConfigChanged, []byte(agentID)) v, err := kv.GetValue(model2.KVAgentIngestConfigChanged, []byte(agentID))
if err != nil { if err != nil {
log.Error(err) log.Error(err)
} }
@ -248,11 +249,11 @@ func (sm *StateManager) syncIngestSettings(agentID string) {
} }
err = sm.agentClient.SaveIngestConfig(context.Background(), ag.GetEndpoint()) err = sm.agentClient.SaveIngestConfig(context.Background(), ag.GetEndpoint())
if err == nil { if err == nil {
kv.AddValue(model.KVAgentIngestConfigChanged,[]byte(agentID), []byte("0")) kv.AddValue(model2.KVAgentIngestConfigChanged,[]byte(agentID), []byte("0"))
} }
} }
func (sm *StateManager) getAvailableAgent(clusterID string) (*agent.Instance, error) { func (sm *StateManager) getAvailableAgent(clusterID string) (*model.Instance, error) {
agents, err := common.LoadAgentsFromES(clusterID) agents, err := common.LoadAgentsFromES(clusterID)
if err != nil { if err != nil {
return nil, err return nil, err
@ -289,14 +290,14 @@ func (sm *StateManager) Stop() {
<-sm.stopCompleteC <-sm.stopCompleteC
} }
func (sm *StateManager) GetAgent(ID string) (*agent.Instance, error) { func (sm *StateManager) GetAgent(ID string) (*model.Instance, error) {
buf, err := kv.GetValue(sm.KVKey, []byte(ID)) buf, err := kv.GetValue(sm.KVKey, []byte(ID))
if err != nil { if err != nil {
return nil, err return nil, err
} }
strTime, _ := jsonparser.GetString(buf, "timestamp") strTime, _ := jsonparser.GetString(buf, "timestamp")
timestamp, _ := time.Parse(time.RFC3339, strTime) timestamp, _ := time.Parse(time.RFC3339, strTime)
inst := &agent.Instance{} inst := &model.Instance{}
inst.ID = ID inst.ID = ID
if time.Since(timestamp) > sm.TTL { if time.Since(timestamp) > sm.TTL {
exists, err := orm.Get(inst) exists, err := orm.Get(inst)
@ -317,7 +318,7 @@ func (sm *StateManager) GetAgent(ID string) (*agent.Instance, error) {
return inst, err return inst, err
} }
func (sm *StateManager) UpdateAgent(inst *agent.Instance, syncToES bool) (*agent.Instance, error) { func (sm *StateManager) UpdateAgent(inst *model.Instance, syncToES bool) (*model.Instance, error) {
//inst.Timestamp = time.Now() //inst.Timestamp = time.Now()
err := kv.AddValue(sm.KVKey, []byte(inst.ID), util.MustToJSONBytes(inst)) err := kv.AddValue(sm.KVKey, []byte(inst.ID), util.MustToJSONBytes(inst))
if syncToES { if syncToES {
@ -332,7 +333,7 @@ func (sm *StateManager) UpdateAgent(inst *agent.Instance, syncToES bool) (*agent
return inst, err return inst, err
} }
func (sm *StateManager) GetTaskAgent(clusterID string) (*agent.Instance, error) { func (sm *StateManager) GetTaskAgent(clusterID string) (*model.Instance, error) {
return nil, nil return nil, nil
} }

92
plugin/api/gateway/api.go
View File

@ -5,14 +5,7 @@
package gateway package gateway
import ( import (
"crypto/tls"
"infini.sh/framework/core/api" "infini.sh/framework/core/api"
"infini.sh/framework/core/api/rbac/enum"
"net"
"net/http"
"net/url"
log "github.com/cihub/seelog"
"time"
) )
type GatewayAPI struct { type GatewayAPI struct {
@ -20,47 +13,48 @@ type GatewayAPI struct {
} }
func InitAPI() { func InitAPI() {
gateway:=GatewayAPI{} //gateway:=GatewayAPI{}
api.HandleAPIMethod(api.POST, "/gateway/instance/try_connect", gateway.RequireLogin(gateway.tryConnect)) //api.HandleAPIMethod(api.POST, "/gateway/instance/try_connect", gateway.RequireLogin(gateway.tryConnect))
api.HandleAPIMethod(api.GET, "/gateway/instance/:instance_id", gateway.RequirePermission(gateway.getInstance, enum.PermissionGatewayInstanceRead)) //api.HandleAPIMethod(api.GET, "/gateway/instance/:instance_id", gateway.RequirePermission(gateway.getInstance, enum.PermissionGatewayInstanceRead))
api.HandleAPIMethod(api.POST, "/gateway/instance", gateway.RequirePermission(gateway.createInstance, enum.PermissionGatewayInstanceWrite)) //api.HandleAPIMethod(api.POST, "/gateway/instance", gateway.RequirePermission(gateway.createInstance, enum.PermissionGatewayInstanceWrite))
api.HandleAPIMethod(api.PUT, "/gateway/instance/:instance_id", gateway.RequirePermission(gateway.updateInstance, enum.PermissionGatewayInstanceWrite)) //api.HandleAPIMethod(api.PUT, "/gateway/instance/:instance_id", gateway.RequirePermission(gateway.updateInstance, enum.PermissionGatewayInstanceWrite))
api.HandleAPIMethod(api.DELETE, "/gateway/instance/:instance_id", gateway.RequirePermission(gateway.deleteInstance, enum.PermissionGatewayInstanceWrite)) //api.HandleAPIMethod(api.DELETE, "/gateway/instance/:instance_id", gateway.RequirePermission(gateway.deleteInstance, enum.PermissionGatewayInstanceWrite))
api.HandleAPIMethod(api.GET, "/gateway/instance/_search", gateway.RequirePermission(gateway.searchInstance, enum.PermissionGatewayInstanceRead)) //api.HandleAPIMethod(api.GET, "/gateway/instance/_search", gateway.RequirePermission(gateway.searchInstance, enum.PermissionGatewayInstanceRead))
api.HandleAPIMethod(api.POST, "/gateway/instance/status", gateway.RequirePermission(gateway.getInstanceStatus, enum.PermissionGatewayInstanceRead)) //api.HandleAPIMethod(api.POST, "/gateway/instance/status", gateway.RequirePermission(gateway.getInstanceStatus, enum.PermissionGatewayInstanceRead))
//
//api.HandleAPIMethod(api.POST, "/gateway/instance/:instance_id/_proxy", gateway.RequirePermission(gateway.proxy, enum.PermissionGatewayInstanceRead))
api.HandleAPIMethod(api.POST, "/gateway/instance/:instance_id/_proxy", gateway.RequirePermission(gateway.proxy, enum.PermissionGatewayInstanceRead)) //api.HandleAPIMethod(api.GET, "/_platform/nodes", gateway.getExecutionNodes)
//
api.HandleAPIMethod(api.GET, "/_platform/nodes", gateway.getExecutionNodes) //api.HandleAPIFunc("/ws_proxy", func(w http.ResponseWriter, req *http.Request) {
api.HandleAPIFunc("/ws_proxy", func(w http.ResponseWriter, req *http.Request) { // log.Debug(req.RequestURI)
log.Debug(req.RequestURI) // endpoint := req.URL.Query().Get("endpoint")
endpoint := req.URL.Query().Get("endpoint") // path := req.URL.Query().Get("path")
path := req.URL.Query().Get("path") // var tlsConfig = &tls.Config{
var tlsConfig = &tls.Config{ // InsecureSkipVerify: true,
InsecureSkipVerify: true, // }
} // target, err := url.Parse(endpoint)
target, err := url.Parse(endpoint) // if err != nil {
if err != nil { // log.Error(err)
log.Error(err) // return
return // }
} // newURL, err := url.Parse(path)
newURL, err := url.Parse(path) // if err != nil {
if err != nil { // log.Error(err)
log.Error(err) // return
return // }
} // req.URL.Path = newURL.Path
req.URL.Path = newURL.Path // req.URL.RawPath = newURL.RawPath
req.URL.RawPath = newURL.RawPath // req.URL.RawQuery = ""
req.URL.RawQuery = "" // req.RequestURI = req.URL.RequestURI()
req.RequestURI = req.URL.RequestURI() // req.Header.Set("HOST", target.Host)
req.Header.Set("HOST", target.Host) // req.Host = target.Host
req.Host = target.Host // wsProxy := NewSingleHostReverseProxy(target)
wsProxy := NewSingleHostReverseProxy(target) // wsProxy.Dial = (&net.Dialer{
wsProxy.Dial = (&net.Dialer{ // Timeout: 30 * time.Second,
Timeout: 30 * time.Second, // KeepAlive: 30 * time.Second,
KeepAlive: 30 * time.Second, // }).Dial
}).Dial // wsProxy.TLSClientConfig = tlsConfig
wsProxy.TLSClientConfig = tlsConfig // wsProxy.ServeHTTP(w, req)
wsProxy.ServeHTTP(w, req) //})
})
} }

946
plugin/api/gateway/instance.go
View File

@ -1,495 +1,455 @@
/* Copyright © INFINI Ltd. All rights reserved. ///* Copyright © INFINI Ltd. All rights reserved.
* web: https://infinilabs.com // * web: https://infinilabs.com
* mail: hello#infini.ltd */ // * mail: hello#infini.ltd */
//
package gateway package gateway
//
//import (
// "fmt"
// log "github.com/cihub/seelog"
// "github.com/segmentio/encoding/json"
// "infini.sh/console/model"
// httprouter "infini.sh/framework/core/api/router"
// elastic2 "infini.sh/framework/core/elastic"
// "infini.sh/framework/core/model"
// "infini.sh/framework/core/orm"
// "infini.sh/framework/core/proxy"
// "infini.sh/framework/core/task"
// "infini.sh/framework/core/util"
// "infini.sh/framework/modules/elastic"
// "net/http"
// "net/url"
// "strconv"
// "strings"
// "time"
//)
//
import ( //func (h *GatewayAPI) getInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
"fmt" // id := ps.MustGetParameter("instance_id")
log "github.com/cihub/seelog" //
"github.com/segmentio/encoding/json" // obj := model.Instance{}
"infini.sh/console/model" // obj.ID = id
"infini.sh/framework/core/agent" //
httprouter "infini.sh/framework/core/api/router" // exists, err := orm.Get(&obj)
elastic2 "infini.sh/framework/core/elastic" // if !exists || err != nil {
"infini.sh/framework/core/orm" // h.WriteJSON(w, util.MapStr{
"infini.sh/framework/core/proxy" // "_id": id,
"infini.sh/framework/core/task" // "found": false,
"infini.sh/framework/core/util" // }, http.StatusNotFound)
"infini.sh/framework/modules/elastic" // return
"net/http" // }
"net/url" // if err != nil {
"strconv" // h.WriteError(w, err.Error(), http.StatusInternalServerError)
"strings" // log.Error(err)
"time" // return
) // }
//
func (h *GatewayAPI) createInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { // h.WriteJSON(w, util.MapStr{
var obj = &model.Instance{} // "found": true,
err := h.DecodeJSON(req, obj) // "_id": id,
if err != nil { // "_source": obj,
h.WriteError(w, err.Error(), http.StatusInternalServerError) // }, 200)
log.Error(err) //}
return //
} //func (h *GatewayAPI) updateInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
// id := ps.MustGetParameter("instance_id")
res, err := h.doConnect(obj.Endpoint, obj.BasicAuth) // obj := model.Instance{}
if err != nil { //
h.WriteError(w, err.Error(), http.StatusInternalServerError) // obj.ID = id
log.Error(err) // exists, err := orm.Get(&obj)
return // if !exists || err != nil {
} // h.WriteJSON(w, util.MapStr{
obj.ID = res.ID // "_id": id,
// "result": "not_found",
exists, err := orm.Get(obj) // }, http.StatusNotFound)
if err != nil && err != elastic.ErrNotFound { // return
h.WriteError(w, err.Error(), http.StatusInternalServerError) // }
log.Error(err) //
return // id = obj.ID
} // create := obj.Created
if exists { // obj = model.Instance{}
h.WriteError(w, "gateway instance already registered", http.StatusInternalServerError) // err = h.DecodeJSON(req, &obj)
return // if err != nil {
} // h.WriteError(w, err.Error(), http.StatusInternalServerError)
err = orm.Create(nil, obj) // log.Error(err)
if err != nil { // return
h.WriteError(w, err.Error(), http.StatusInternalServerError) // }
log.Error(err) //
return // //protect
} // obj.ID = id
// obj.Created = create
h.WriteJSON(w, util.MapStr{ // err = orm.Update(nil, &obj)
"_id": obj.ID, // if err != nil {
"result": "created", // h.WriteError(w, err.Error(), http.StatusInternalServerError)
}, 200) // log.Error(err)
// return
} // }
//
func (h *GatewayAPI) getInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { // h.WriteJSON(w, util.MapStr{
id := ps.MustGetParameter("instance_id") // "_id": obj.ID,
// "result": "updated",
obj := model.Instance{} // }, 200)
obj.ID = id //}
//
exists, err := orm.Get(&obj) //func (h *GatewayAPI) deleteInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
if !exists || err != nil { // id := ps.MustGetParameter("instance_id")
h.WriteJSON(w, util.MapStr{ //
"_id": id, // obj := model.Instance{}
"found": false, // obj.ID = id
}, http.StatusNotFound) //
return // exists, err := orm.Get(&obj)
} // if !exists || err != nil {
if err != nil { // h.WriteJSON(w, util.MapStr{
h.WriteError(w, err.Error(), http.StatusInternalServerError) // "_id": id,
log.Error(err) // "result": "not_found",
return // }, http.StatusNotFound)
} // return
// }
h.WriteJSON(w, util.MapStr{ //
"found": true, // //check reference
"_id": id, // query := util.MapStr{
"_source": obj, // "size": 1,
}, 200) // "query": util.MapStr{
} // "bool": util.MapStr{
// "must": []util.MapStr{
func (h *GatewayAPI) updateInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { // {
id := ps.MustGetParameter("instance_id") // "term": util.MapStr{
obj := model.Instance{} // "metadata.labels.permit_nodes.id": util.MapStr{
// "value": id,
obj.ID = id // },
exists, err := orm.Get(&obj) // },
if !exists || err != nil { // },
h.WriteJSON(w, util.MapStr{ // {
"_id": id, // "terms": util.MapStr{
"result": "not_found", // "metadata.type": []string{"cluster_migration", "cluster_comparison"},
}, http.StatusNotFound) // },
return // },
} // },
// "must_not": []util.MapStr{
id = obj.ID // {
create := obj.Created // "terms": util.MapStr{
obj = model.Instance{} // "status": []string{task.StatusError, task.StatusComplete},
err = h.DecodeJSON(req, &obj) // },
if err != nil { // },
h.WriteError(w, err.Error(), http.StatusInternalServerError) // },
log.Error(err) // },
return // },
} // }
// q := &orm.Query{
//protect // RawQuery: util.MustToJSONBytes(query),
obj.ID = id // }
obj.Created = create // err, result := orm.Search(task.Task{}, q)
err = orm.Update(nil, &obj) // if err != nil {
if err != nil { // h.WriteError(w, err.Error(), http.StatusInternalServerError)
h.WriteError(w, err.Error(), http.StatusInternalServerError) // log.Error(err)
log.Error(err) // return
return // }
} // if len(result.Result) > 0 {
// var taskId interface{}
h.WriteJSON(w, util.MapStr{ // if m, ok := result.Result[0].(map[string]interface{}); ok {
"_id": obj.ID, // taskId = m["id"]
"result": "updated", // }
}, 200) // h.WriteError(w, fmt.Sprintf("failed to delete gateway instance [%s] since it is used by task [%v]", id, taskId), http.StatusInternalServerError)
} // return
// }
func (h *GatewayAPI) deleteInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { //
id := ps.MustGetParameter("instance_id") // err = orm.Delete(nil, &obj)
// if err != nil {
obj := model.Instance{} // h.WriteError(w, err.Error(), http.StatusInternalServerError)
obj.ID = id // log.Error(err)
// return
exists, err := orm.Get(&obj) // }
if !exists || err != nil { //
h.WriteJSON(w, util.MapStr{ // h.WriteJSON(w, util.MapStr{
"_id": id, // "_id": obj.ID,
"result": "not_found", // "result": "deleted",
}, http.StatusNotFound) // }, 200)
return //}
} //
//func (h *GatewayAPI) searchInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
//check reference //
query := util.MapStr{ // var (
"size": 1, // keyword = h.GetParameterOrDefault(req, "keyword", "")
"query": util.MapStr{ // queryDSL = `{"query":{"bool":{"must":[%s]}}, "size": %d, "from": %d}`
"bool": util.MapStr{ // strSize = h.GetParameterOrDefault(req, "size", "20")
"must": []util.MapStr{ // strFrom = h.GetParameterOrDefault(req, "from", "0")
{ // mustBuilder = &strings.Builder{}
"term": util.MapStr{ // )
"metadata.labels.permit_nodes.id": util.MapStr{ // if keyword != "" {
"value": id, // mustBuilder.WriteString(fmt.Sprintf(`{"query_string":{"default_field":"*","query": "%s"}}`, keyword))
}, // }
}, // size, _ := strconv.Atoi(strSize)
}, // if size <= 0 {
{ // size = 20
"terms": util.MapStr{ // }
"metadata.type": []string{"cluster_migration", "cluster_comparison"}, // from, _ := strconv.Atoi(strFrom)
}, // if from < 0 {
}, // from = 0
}, // }
"must_not": []util.MapStr{ //
{ // q := orm.Query{}
"terms": util.MapStr{ // queryDSL = fmt.Sprintf(queryDSL, mustBuilder.String(), size, from)
"status": []string{task.StatusError, task.StatusComplete}, // q.RawQuery = []byte(queryDSL)
}, //
}, // err, res := orm.Search(&model.Instance{}, &q)
}, // if err != nil {
}, // log.Error(err)
}, // h.WriteError(w, err.Error(), http.StatusInternalServerError)
} // return
q := &orm.Query{ // }
RawQuery: util.MustToJSONBytes(query), // h.Write(w, res.Raw)
} //}
err, result := orm.Search(task.Task{}, q) //
if err != nil { //func (h *GatewayAPI) getInstanceStatus(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
h.WriteError(w, err.Error(), http.StatusInternalServerError) // var instanceIDs = []string{}
log.Error(err) // err := h.DecodeJSON(req, &instanceIDs)
return // if err != nil {
} // log.Error(err)
if len(result.Result) > 0 { // h.WriteError(w, err.Error(), http.StatusInternalServerError)
var taskId interface{} // return
if m, ok := result.Result[0].(map[string]interface{}); ok { // }
taskId = m["id"] // if len(instanceIDs) == 0 {
} // h.WriteJSON(w, util.MapStr{}, http.StatusOK)
h.WriteError(w, fmt.Sprintf("failed to delete gateway instance [%s] since it is used by task [%v]", id, taskId), http.StatusInternalServerError) // return
return // }
} // q := orm.Query{}
// queryDSL := util.MapStr{
err = orm.Delete(nil, &obj) // "query": util.MapStr{
if err != nil { // "terms": util.MapStr{
h.WriteError(w, err.Error(), http.StatusInternalServerError) // "_id": instanceIDs,
log.Error(err) // },
return // },
} // }
// q.RawQuery = util.MustToJSONBytes(queryDSL)
h.WriteJSON(w, util.MapStr{ //
"_id": obj.ID, // err, res := orm.Search(&model.Instance{}, &q)
"result": "deleted", // if err != nil {
}, 200) // log.Error(err)
} // h.WriteError(w, err.Error(), http.StatusInternalServerError)
// return
func (h *GatewayAPI) searchInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { // }
// result := util.MapStr{}
var ( // for _, item := range res.Result {
keyword = h.GetParameterOrDefault(req, "keyword", "") // instance := util.MapStr(item.(map[string]interface{}))
queryDSL = `{"query":{"bool":{"must":[%s]}}, "size": %d, "from": %d}` // if err != nil {
strSize = h.GetParameterOrDefault(req, "size", "20") // log.Error(err)
strFrom = h.GetParameterOrDefault(req, "from", "0") // continue
mustBuilder = &strings.Builder{} // }
) // endpoint, _ := instance.GetValue("endpoint")
if keyword != "" { // username, _ := instance.GetValue("basic_auth.username")
mustBuilder.WriteString(fmt.Sprintf(`{"query_string":{"default_field":"*","query": "%s"}}`, keyword)) // if username == nil {
} // username = ""
size, _ := strconv.Atoi(strSize) // }
if size <= 0 { // password, _ := instance.GetValue("basic_auth.password")
size = 20 // if password == nil {
} // password = ""
from, _ := strconv.Atoi(strFrom) // }
if from < 0 { // gid, _ := instance.GetValue("id")
from = 0 // res, err := proxy.DoProxyRequest(&proxy.Request{
} // Endpoint: endpoint.(string),
// Method: http.MethodGet,
q := orm.Query{} // Path: "/stats",
queryDSL = fmt.Sprintf(queryDSL, mustBuilder.String(), size, from) // BasicAuth: instance.BasicAuth{
q.RawQuery = []byte(queryDSL) // Username: username.(string),
// Password: password.(string),
err, res := orm.Search(&model.Instance{}, &q) // },
if err != nil { // })
log.Error(err) // if err != nil {
h.WriteError(w, err.Error(), http.StatusInternalServerError) // log.Error(err)
return // result[gid.(string)] = util.MapStr{}
} // continue
h.Write(w, res.Raw) // }
} // var resMap = util.MapStr{}
// err = util.FromJSONBytes(res.Body, &resMap)
func (h *GatewayAPI) getInstanceStatus(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { // if err != nil {
var instanceIDs = []string{} // result[gid.(string)] = util.MapStr{}
err := h.DecodeJSON(req, &instanceIDs) // log.Errorf("get stats of %v error: %v", endpoint, err)
if err != nil { // continue
log.Error(err) // }
h.WriteError(w, err.Error(), http.StatusInternalServerError) //
return // result[gid.(string)] = resMap
} // }
if len(instanceIDs) == 0 { // h.WriteJSON(w, result, http.StatusOK)
h.WriteJSON(w, util.MapStr{}, http.StatusOK) //}
return //func (h *GatewayAPI) proxy(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
} // var (
q := orm.Query{} // method = h.Get(req, "method", "GET")
queryDSL := util.MapStr{ // path = h.Get(req, "path", "")
"query": util.MapStr{ // )
"terms": util.MapStr{ // instanceID := ps.MustGetParameter("instance_id")
"_id": instanceIDs, //
}, // obj := model.Instance{}
}, // obj.ID = instanceID
} //
q.RawQuery = util.MustToJSONBytes(queryDSL) // exists, err := orm.Get(&obj)
// if err != nil {
err, res := orm.Search(&model.Instance{}, &q) // h.WriteError(w, err.Error(), http.StatusInternalServerError)
if err != nil { // log.Error(err)
log.Error(err) // return
h.WriteError(w, err.Error(), http.StatusInternalServerError) // }
return // if !exists {
} // h.WriteJSON(w, util.MapStr{
result := util.MapStr{} // "error": "gateway instance not found",
for _, item := range res.Result { // }, http.StatusNotFound)
instance := util.MapStr(item.(map[string]interface{})) // return
if err != nil { // }
log.Error(err) // res, err := proxy.DoProxyRequest(&proxy.Request{
continue // Method: method,
} // Endpoint: obj.Endpoint,
endpoint, _ := instance.GetValue("endpoint") // Path: path,
username, _ := instance.GetValue("basic_auth.username") // Body: req.Body,
if username == nil { // BasicAuth: obj.BasicAuth,
username = "" // ContentLength: int(req.ContentLength),
} // })
password, _ := instance.GetValue("basic_auth.password") // if err != nil {
if password == nil { // h.WriteError(w, err.Error(), http.StatusInternalServerError)
password = "" // log.Error(err)
} // return
gid, _ := instance.GetValue("id") // }
res, err := proxy.DoProxyRequest(&proxy.Request{ // h.WriteHeader(w, res.StatusCode)
Endpoint: endpoint.(string), // h.Write(w, res.Body)
Method: http.MethodGet, //}
Path: "/stats", //
BasicAuth: agent.BasicAuth{ //type GatewayConnectResponse struct {
Username: username.(string), // ID string `json:"id"`
Password: password.(string), // Name string `json:"name"`
}, // Tagline string `json:"tagline"`
}) // Version struct {
if err != nil { // BuildDate string `json:"build_date"`
log.Error(err) // BuildHash string `json:"build_hash"`
result[gid.(string)] = util.MapStr{} // EOLDate string `json:"eol_date"`
continue // Number string `json:"number"`
} // } `json:"version"`
var resMap = util.MapStr{} //
err = util.FromJSONBytes(res.Body, &resMap) //}
if err != nil { //func (h *GatewayAPI) doConnect(endpoint string, basicAuth model.BasicAuth) (*GatewayConnectResponse, error) {
result[gid.(string)] = util.MapStr{} // res, err := proxy.DoProxyRequest(&proxy.Request{
log.Errorf("get stats of %v error: %v", endpoint, err) // Method: http.MethodGet,
continue // Endpoint: endpoint,
} // Path: "/_info",
// BasicAuth: basicAuth,
result[gid.(string)] = resMap // })
} // if err != nil {
h.WriteJSON(w, result, http.StatusOK) // return nil, err
} // }
func (h *GatewayAPI) proxy(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { // if res.StatusCode == http.StatusNotFound {
var ( // return nil, fmt.Errorf("unknow gateway version")
method = h.Get(req, "method", "GET") // }
path = h.Get(req, "path", "") // b := res.Body
) // gres := &GatewayConnectResponse{}
instanceID := ps.MustGetParameter("instance_id") // err = json.Unmarshal(b, gres)
// return gres, err
obj := model.Instance{} //
obj.ID = instanceID //}
//
exists, err := orm.Get(&obj) //func (h *GatewayAPI) tryConnect(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
if err != nil { // var reqBody = struct {
h.WriteError(w, err.Error(), http.StatusInternalServerError) // Endpoint string `json:"endpoint"`
log.Error(err) // BasicAuth model.BasicAuth
return // }{}
} // err := h.DecodeJSON(req, &reqBody)
if !exists { // if err != nil {
h.WriteJSON(w, util.MapStr{ // h.WriteError(w, err.Error(), http.StatusInternalServerError)
"error": "gateway instance not found", // return
}, http.StatusNotFound) // }
return // connectRes, err := h.doConnect(reqBody.Endpoint, reqBody.BasicAuth)
} // if err != nil {
res, err := proxy.DoProxyRequest(&proxy.Request{ // h.WriteError(w, err.Error(), http.StatusInternalServerError)
Method: method, // return
Endpoint: obj.Endpoint, // }
Path: path, // h.WriteJSON(w, connectRes, http.StatusOK)
Body: req.Body, //}
BasicAuth: obj.BasicAuth, //
ContentLength: int(req.ContentLength), //func (h *GatewayAPI) getExecutionNodes(w http.ResponseWriter, req *http.Request, ps httprouter.Params){
}) // var (
if err != nil { // keyword = h.GetParameterOrDefault(req, "keyword", "")
h.WriteError(w, err.Error(), http.StatusInternalServerError) // strSize = h.GetParameterOrDefault(req, "size", "10")
log.Error(err) // strFrom = h.GetParameterOrDefault(req, "from", "0")
return // )
} // size, _ := strconv.Atoi(strSize)
h.WriteHeader(w, res.StatusCode) // if size <= 0 {
h.Write(w, res.Body) // size = 10
} // }
// from, _ := strconv.Atoi(strFrom)
type GatewayConnectResponse struct { // if from < 0 {
ID string `json:"id"` // from = 0
Name string `json:"name"` // }
Tagline string `json:"tagline"` // gatewayIndexName := orm.GetIndexName(model.Instance{})
Version struct { //
BuildDate string `json:"build_date"` // query := util.MapStr{
BuildHash string `json:"build_hash"` // "size": size,
EOLDate string `json:"eol_date"` // "from": from,
Number string `json:"number"` // "sort": []util.MapStr{
} `json:"version"` // {
// "created": util.MapStr{
} // "order": "desc",
func (h *GatewayAPI) doConnect(endpoint string, basicAuth agent.BasicAuth) (*GatewayConnectResponse, error) { // },
res, err := proxy.DoProxyRequest(&proxy.Request{ // },
Method: http.MethodGet, // },
Endpoint: endpoint, // }
Path: "/_framework/api/_info", // if keyword != "" {
BasicAuth: basicAuth, // query["query"] = util.MapStr{
}) // "bool": util.MapStr{
if err != nil { // "must": []util.MapStr{
return nil, err // {
} // "prefix": util.MapStr{
if res.StatusCode == http.StatusNotFound { // "name": util.MapStr{
return nil, fmt.Errorf("unknow gateway version") // "value": keyword,
} // },
b := res.Body // },
gres := &GatewayConnectResponse{} // },
err = json.Unmarshal(b, gres) // },
return gres, err // },
// }
} // }
// q := orm.Query{
func (h *GatewayAPI) tryConnect(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { // IndexName: gatewayIndexName,
var reqBody = struct { // RawQuery: util.MustToJSONBytes(query),
Endpoint string `json:"endpoint"` // }
BasicAuth agent.BasicAuth // err, result := orm.Search(nil, &q)
}{} // if err != nil {
err := h.DecodeJSON(req, &reqBody) // h.WriteError(w, err.Error(), http.StatusInternalServerError)
if err != nil { // return
h.WriteError(w, err.Error(), http.StatusInternalServerError) // }
return //
} // searchRes := elastic2.SearchResponse{}
connectRes, err := h.doConnect(reqBody.Endpoint, reqBody.BasicAuth) // err = util.FromJSONBytes(result.Raw, &searchRes)
if err != nil { // if err != nil||searchRes.ESError!=nil {
h.WriteError(w, err.Error(), http.StatusInternalServerError) // msg:=fmt.Sprintf("%v,%v",err,searchRes.ESError)
return // h.WriteError(w, msg, http.StatusInternalServerError)
} // return
h.WriteJSON(w, connectRes, http.StatusOK) // }
} // var nodes = []util.MapStr{}
//
func (h *GatewayAPI) getExecutionNodes(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ // for _, hit := range searchRes.Hits.Hits {
var ( // buf := util.MustToJSONBytes(hit.Source)
keyword = h.GetParameterOrDefault(req, "keyword", "") // inst := model.Instance{}
strSize = h.GetParameterOrDefault(req, "size", "10") // err = util.FromJSONBytes(buf, &inst)
strFrom = h.GetParameterOrDefault(req, "from", "0") // if err != nil {
) // log.Error(err)
size, _ := strconv.Atoi(strSize) // continue
if size <= 0 { // }
size = 10 // node := util.MapStr{
} // "id": inst.ID,
from, _ := strconv.Atoi(strFrom) // "name": inst.Name,
if from < 0 { // "available": false,
from = 0 // "type": "gateway",
} // }
gatewayIndexName := orm.GetIndexName(model.Instance{}) // ul, err := url.Parse(inst.Endpoint)
// if err != nil {
query := util.MapStr{ // log.Error(err)
"size": size, // continue
"from": from, // }
"sort": []util.MapStr{ // node["host"] = ul.Host
{ // err = inst.TryConnectWithTimeout(time.Second)
"created": util.MapStr{ // if err != nil {
"order": "desc", // log.Error(err)
}, // }else{
}, // node["available"] = true
}, // }
} //
if keyword != "" { // nodes = append(nodes, node)
query["query"] = util.MapStr{ // }
"bool": util.MapStr{ // h.WriteJSON(w, nodes, http.StatusOK)
"must": []util.MapStr{ //}
{
"prefix": util.MapStr{
"name": util.MapStr{
"value": keyword,
},
},
},
},
},
}
}
q := orm.Query{
IndexName: gatewayIndexName,
RawQuery: util.MustToJSONBytes(query),
}
err, result := orm.Search(nil, &q)
if err != nil {
h.WriteError(w, err.Error(), http.StatusInternalServerError)
return
}
searchRes := elastic2.SearchResponse{}
err = util.FromJSONBytes(result.Raw, &searchRes)
if err != nil||searchRes.ESError!=nil {
msg:=fmt.Sprintf("%v,%v",err,searchRes.ESError)
h.WriteError(w, msg, http.StatusInternalServerError)
return
}
var nodes = []util.MapStr{}
for _, hit := range searchRes.Hits.Hits {
buf := util.MustToJSONBytes(hit.Source)
inst := model.Instance{}
err = util.FromJSONBytes(buf, &inst)
if err != nil {
log.Error(err)
continue
}
node := util.MapStr{
"id": inst.ID,
"name": inst.Name,
"available": false,
"type": "gateway",
}
ul, err := url.Parse(inst.Endpoint)
if err != nil {
log.Error(err)
continue
}
node["host"] = ul.Host
err = inst.TryConnectWithTimeout(time.Second)
if err != nil {
log.Error(err)
}else{
node["available"] = true
}
nodes = append(nodes, node)
}
h.WriteJSON(w, nodes, http.StatusOK)
}

186
plugin/api/gateway/websocket_proxy.go
View File

@ -1,186 +0,0 @@
/* Copyright © INFINI Ltd. All rights reserved.
* Web: https://infinilabs.com
* Email: hello#infini.ltd */
package gateway
import (
"crypto/tls"
"io"
log "github.com/cihub/seelog"
"net"
"net/http"
"net/url"
"strings"
)
// ReverseProxy is a WebSocket reverse proxy. It will not work with a regular
// HTTP request, so it is the caller's responsiblity to ensure the incoming
// request is a WebSocket request.
type ReverseProxy struct {
// Director must be a function which modifies
// the request into a new request to be sent
// using Transport. Its response is then copied
// back to the original client unmodified.
Director func(*http.Request)
// Dial specifies the dial function for dialing the proxied
// server over tcp.
// If Dial is nil, net.Dial is used.
Dial func(network, addr string) (net.Conn, error)
// TLSClientConfig specifies the TLS configuration to use for 'wss'.
// If nil, the default configuration is used.
TLSClientConfig *tls.Config
// ErrorLog specifies an optional logger for errors
// that occur when attempting to proxy the request.
// If nil, logging goes to os.Stderr via the log package's
// standard logger.
ErrorLog *log.LoggerInterface
}
// stolen from net/http/httputil. singleJoiningSlash ensures that the route
// '/a/' joined with '/b' becomes '/a/b'.
func singleJoiningSlash(a, b string) string {
aslash := strings.HasSuffix(a, "/")
bslash := strings.HasPrefix(b, "/")
switch {
case aslash && bslash:
return a + b[1:]
case !aslash && !bslash:
return a + "/" + b
}
return a + b
}
// NewSingleHostReverseProxy returns a new websocket ReverseProxy. The path
// rewrites follow the same rules as the httputil.ReverseProxy. If the target
// url has the path '/foo' and the incoming request '/bar', the request path
// will be updated to '/foo/bar' before forwarding.
// Scheme should specify if 'ws' or 'wss' should be used.
func NewSingleHostReverseProxy(target *url.URL) *ReverseProxy {
targetQuery := target.RawQuery
director := func(req *http.Request) {
req.URL.Scheme = target.Scheme
req.URL.Host = target.Host
req.URL.Path = singleJoiningSlash(target.Path, req.URL.Path)
if targetQuery == "" || req.URL.RawQuery == "" {
req.URL.RawQuery = targetQuery + req.URL.RawQuery
} else {
req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery
}
}
return &ReverseProxy{Director: director}
}
// Function to implement the http.Handler interface.
func (p *ReverseProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
logFunc := log.Errorf
if !IsWebSocketRequest(r) {
http.Error(w, "Cannot handle non-WebSocket requests", 500)
logFunc("Received a request that was not a WebSocket request")
return
}
outreq := new(http.Request)
// shallow copying
*outreq = *r
p.Director(outreq)
host := outreq.URL.Host
if clientIP, _, err := net.SplitHostPort(r.RemoteAddr); err == nil {
// If we aren't the first proxy retain prior
// X-Forwarded-For information as a comma+space
// separated list and fold multiple headers into one.
if prior, ok := outreq.Header["X-Forwarded-For"]; ok {
clientIP = strings.Join(prior, ", ") + ", " + clientIP
}
outreq.Header.Set("X-Forwarded-For", clientIP)
}
dial := p.Dial
if dial == nil {
dial = net.Dial
}
// if host does not specify a port, use the default http port
if !strings.Contains(host, ":") {
if outreq.URL.Scheme == "wss" {
host = host + ":443"
} else {
host = host + ":80"
}
}
if outreq.URL.Scheme == "wss" {
var tlsConfig *tls.Config
if p.TLSClientConfig == nil {
tlsConfig = &tls.Config{}
} else {
tlsConfig = p.TLSClientConfig
}
dial = func(network, address string) (net.Conn, error) {
return tls.Dial("tcp", host, tlsConfig)
}
}
d, err := dial("tcp", host)
if err != nil {
http.Error(w, "Error forwarding request.", 500)
logFunc("Error dialing websocket backend %s: %v", outreq.URL, err)
return
}
// All request generated by the http package implement this interface.
hj, ok := w.(http.Hijacker)
if !ok {
http.Error(w, "Not a hijacker?", 500)
return
}
// Hijack() tells the http package not to do anything else with the connection.
// After, it bcomes this functions job to manage it. `nc` is of type *net.Conn.
nc, _, err := hj.Hijack()
if err != nil {
logFunc("Hijack error: %v", err)
return
}
defer nc.Close() // must close the underlying net connection after hijacking
defer d.Close()
// write the modified incoming request to the dialed connection
err = outreq.Write(d)
if err != nil {
logFunc("Error copying request to target: %v", err)
return
}
errc := make(chan error, 2)
cp := func(dst io.Writer, src io.Reader) {
_, err := io.Copy(dst, src)
errc <- err
}
go cp(d, nc)
go cp(nc, d)
<-errc
}
// IsWebSocketRequest returns a boolean indicating whether the request has the
// headers of a WebSocket handshake request.
func IsWebSocketRequest(r *http.Request) bool {
contains := func(key, val string) bool {
vv := strings.Split(r.Header.Get(key), ",")
for _, v := range vv {
if val == strings.ToLower(strings.TrimSpace(v)) {
return true
}
}
return false
}
if !contains("Connection", "upgrade") {
return false
}
if !contains("Upgrade", "websocket") {
return false
}
return true
}

6
plugin/setup/setup.go
View File

@ -495,10 +495,6 @@ func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps http
} }
} }
if err != nil {
panic(err)
}
//处理索引 //处理索引
elastic2.InitSchema() elastic2.InitSchema()
//init security //init security
@ -573,7 +569,7 @@ func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps http
_, err = util.FilePutContent(file, fmt.Sprintf("configs.template:\n - name: \"system\"\n path: ./config/system_config.tpl\n variable:\n "+ _, err = util.FilePutContent(file, fmt.Sprintf("configs.template:\n - name: \"system\"\n path: ./config/system_config.tpl\n variable:\n "+
"CLUSTER_ID: %v\n CLUSTER_ENDPINT: \"%v\"\n "+ "CLUSTER_ID: %v\n CLUSTER_ENDPINT: \"%v\"\n "+
"CLUSTER_USER: \"%v\"\n CLUSTER_VER: \"%v\"\n CLUSTER_DISTRIBUTION: \"%v\"\n INDEX_PREFIX: \"%v\"", "CLUSTER_USER: \"%v\"\n CLUSTER_VER: \"%v\"\n CLUSTER_DISTRIBUTION: \"%v\"\n INDEX_PREFIX: \"%v\"",
GlobalSystemElasticsearchID, cfg.Endpoint, cfg.BasicAuth.Username, cfg.Version, cfg.Distribution, cfg1.IndexPrefix)) GlobalSystemElasticsearchID, cfg.GetAnyEndpoint(), cfg.BasicAuth.Username, cfg.Version, cfg.Distribution, cfg1.IndexPrefix))
if err != nil { if err != nil {
panic(err) panic(err)
} }

4
plugin/task_manager/common_api.go
View File

@ -3,6 +3,7 @@ package task_manager
import ( import (
"errors" "errors"
"fmt" "fmt"
model2 "infini.sh/console/model"
"net/http" "net/http"
"strconv" "strconv"
"strings" "strings"
@ -10,7 +11,6 @@ import (
log "github.com/cihub/seelog" log "github.com/cihub/seelog"
"infini.sh/console/model"
migration_util "infini.sh/console/plugin/task_manager/util" migration_util "infini.sh/console/plugin/task_manager/util"
httprouter "infini.sh/framework/core/api/router" httprouter "infini.sh/framework/core/api/router"
@ -560,7 +560,7 @@ func (h *APIHandler) getChildPipelineInfosFromGateway(pipelineTaskIDs map[string
var err error var err error
for instID, taskIDs := range pipelineTaskIDs { for instID, taskIDs := range pipelineTaskIDs {
inst := &model.Instance{} inst := &model2.TaskWorker{}
inst.ID = instID inst.ID = instID
_, err = orm.Get(inst) _, err = orm.Get(inst)
if err != nil { if err != nil {

5
plugin/task_manager/model/scheduler.go
View File

@ -2,13 +2,12 @@ package model
import ( import (
"errors" "errors"
"infini.sh/console/model" "infini.sh/console/model"
) )
type Scheduler interface { type Scheduler interface {
GetPreferenceInstance(config ExecutionConfig) (instance *model.Instance, err error) GetPreferenceInstance(config ExecutionConfig) (instance *model.TaskWorker, err error)
GetInstance(instanceID string) (instance *model.Instance, err error) GetInstance(instanceID string) (instance *model.TaskWorker, err error)
IncrInstanceJobs(instanceID string) IncrInstanceJobs(instanceID string)
DecrInstanceJobs(instanceID string) DecrInstanceJobs(instanceID string)
RefreshInstanceJobsFromES() error RefreshInstanceJobsFromES() error

6
plugin/task_manager/pipeline_task/pipeline_task.go
View File

@ -3,13 +3,13 @@ package pipeline_task
import ( import (
"errors" "errors"
"fmt" "fmt"
"infini.sh/console/model"
"strconv" "strconv"
"strings" "strings"
"time" "time"
log "github.com/cihub/seelog" log "github.com/cihub/seelog"
"infini.sh/console/model"
migration_model "infini.sh/console/plugin/task_manager/model" migration_model "infini.sh/console/plugin/task_manager/model"
migration_util "infini.sh/console/plugin/task_manager/util" migration_util "infini.sh/console/plugin/task_manager/util"
@ -169,7 +169,7 @@ func (p *processor) handlePendingStopPipelineTask(taskItem *task.Task) error {
return nil return nil
} }
func (p *processor) cleanGatewayPipeline(taskItem *task.Task) (instance *model.Instance, err error) { func (p *processor) cleanGatewayPipeline(taskItem *task.Task) (instance *model.TaskWorker, err error) {
instance, err = p.getPipelineExecutionInstance(taskItem) instance, err = p.getPipelineExecutionInstance(taskItem)
if err != nil { if err != nil {
log.Errorf("failed to get execution instance for task [%s], err: %v", taskItem.ID, err) log.Errorf("failed to get execution instance for task [%s], err: %v", taskItem.ID, err)
@ -184,7 +184,7 @@ func (p *processor) cleanGatewayPipeline(taskItem *task.Task) (instance *model.I
return instance, nil return instance, nil
} }
func (p *processor) getPipelineExecutionInstance(taskItem *task.Task) (*model.Instance, error) { func (p *processor) getPipelineExecutionInstance(taskItem *task.Task) (*model.TaskWorker, error) {
instanceID, _ := util.ExtractString(taskItem.Metadata.Labels["execution_instance_id"]) instanceID, _ := util.ExtractString(taskItem.Metadata.Labels["execution_instance_id"])
instance, err := p.scheduler.GetInstance(instanceID) instance, err := p.scheduler.GetInstance(instanceID)
if err != nil { if err != nil {

14
plugin/task_manager/scheduler/scheduler.go
View File

@ -53,7 +53,7 @@ func NewScheduler(elasticsearch, indexName string, checkInstanceAvailable bool,
return scheduler, nil return scheduler, nil
} }
func (p *scheduler) GetPreferenceInstance(config migration_model.ExecutionConfig) (*model.Instance, error) { func (p *scheduler) GetPreferenceInstance(config migration_model.ExecutionConfig) (*model.TaskWorker, error) {
var ( var (
err error err error
minID string minID string
@ -64,7 +64,7 @@ func (p *scheduler) GetPreferenceInstance(config migration_model.ExecutionConfig
instanceTotal := p.getInstanceState(node.ID).Total instanceTotal := p.getInstanceState(node.ID).Total
if instanceTotal < minTotal { if instanceTotal < minTotal {
if p.CheckInstanceAvailable { if p.CheckInstanceAvailable {
tempInst := model.Instance{} tempInst := model.TaskWorker{}
tempInst.ID = node.ID tempInst.ID = node.ID
_, err = orm.Get(&tempInst) _, err = orm.Get(&tempInst)
if err != nil { if err != nil {
@ -95,11 +95,11 @@ func (p *scheduler) GetPreferenceInstance(config migration_model.ExecutionConfig
return instance, nil return instance, nil
} }
func (p *scheduler) GetInstance(instanceID string) (*model.Instance, error) { func (p *scheduler) GetInstance(instanceID string) (*model.TaskWorker, error) {
if instanceID == "" { if instanceID == "" {
return nil, errors.New("invalid instanceID") return nil, errors.New("invalid instanceID")
} }
instance := model.Instance{} instance := model.TaskWorker{}
instance.ID = instanceID instance.ID = instanceID
_, err := orm.Get(&instance) _, err := orm.Get(&instance)
@ -114,7 +114,7 @@ func (p *scheduler) GetInstance(instanceID string) (*model.Instance, error) {
return &instance, nil return &instance, nil
} }
func (p *scheduler) initializeInstance(instance *model.Instance) error { func (p *scheduler) initializeInstance(instance *model.TaskWorker) error {
lastInitializedAt := p.getLastInitializedAt(instance.ID) lastInitializedAt := p.getLastInitializedAt(instance.ID)
if time.Now().Sub(lastInitializedAt) < initializeInterval { if time.Now().Sub(lastInitializedAt) < initializeInterval {
return nil return nil
@ -162,7 +162,7 @@ func (p *scheduler) initializeInstance(instance *model.Instance) error {
// user could change the following configurations manually: // user could change the following configurations manually:
// - input_queue (metrics.logging_queue) // - input_queue (metrics.logging_queue)
// - elasticsearch (elasticsearch.name) // - elasticsearch (elasticsearch.name)
func (p *scheduler) createPipelineLoggingMerge(instance *model.Instance) error { func (p *scheduler) createPipelineLoggingMerge(instance *model.TaskWorker) error {
cfg := &migration_model.PipelineTaskConfig{ cfg := &migration_model.PipelineTaskConfig{
Name: "pipeline_logging_merge", Name: "pipeline_logging_merge",
AutoStart: true, AutoStart: true,
@ -194,7 +194,7 @@ func (p *scheduler) createPipelineLoggingMerge(instance *model.Instance) error {
return nil return nil
} }
func (p *scheduler) createIngestPipelineLogging(instance *model.Instance) error { func (p *scheduler) createIngestPipelineLogging(instance *model.TaskWorker) error {
cfg := &migration_model.PipelineTaskConfig{ cfg := &migration_model.PipelineTaskConfig{
Name: "ingest_pipeline_logging", Name: "ingest_pipeline_logging",
AutoStart: true, AutoStart: true,

4
service/alerting/env.go
View File

@ -6,8 +6,8 @@ package alerting
import ( import (
"fmt" "fmt"
"infini.sh/framework/core/config"
config2 "infini.sh/console/config" config2 "infini.sh/console/config"
"infini.sh/framework/core/config"
"infini.sh/framework/core/env" "infini.sh/framework/core/env"
"infini.sh/framework/core/global" "infini.sh/framework/core/global"
"infini.sh/framework/core/kv" "infini.sh/framework/core/kv"
@ -53,6 +53,6 @@ func GetInnerConsoleEndpoint() (string, error){
if !ok { if !ok {
return "", fmt.Errorf("web config not exists") return "", fmt.Errorf("web config not exists")
} }
endpoint := fmt.Sprintf("%s://%s", appConfig.GetSchema(), appConfig.Network.GetPublishAddr()) endpoint := appConfig.GetEndpoint()
return endpoint, nil return endpoint, nil
} }