fixed merge master conflicts
This commit is contained in:
commit
3dedda4e62
|
@ -32,7 +32,7 @@ pipeline {
|
||||||
|
|
||||||
sh label: 'copy-license', script: 'cd /home/jenkins/go/src/infini.sh/console && cp ../framework/LICENSE bin && cat ../framework/NOTICE NOTICE > bin/NOTICE'
|
sh label: 'copy-license', script: 'cd /home/jenkins/go/src/infini.sh/console && cp ../framework/LICENSE bin && cat ../framework/NOTICE NOTICE > bin/NOTICE'
|
||||||
|
|
||||||
sh label: 'copy-configs', script: 'cd /home/jenkins/go/src/infini.sh/console && mkdir -p bin/config && cp config/*.json bin/config'
|
sh label: 'copy-configs', script: 'cd /home/jenkins/go/src/infini.sh/console && mkdir -p bin/config && cp config/*.json bin/config && cp config/*.yml bin/config && cp config/*.tpl bin/config'
|
||||||
|
|
||||||
sh label: 'package-linux-amd64', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-amd64.tar.gz console-linux-amd64 console.yml LICENSE NOTICE config'
|
sh label: 'package-linux-amd64', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-amd64.tar.gz console-linux-amd64 console.yml LICENSE NOTICE config'
|
||||||
sh label: 'package-linux-386', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-386.tar.gz console-linux-386 console.yml LICENSE NOTICE config'
|
sh label: 'package-linux-386', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-386.tar.gz console-linux-386 console.yml LICENSE NOTICE config'
|
||||||
|
|
|
@ -8,14 +8,14 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/buger/jsonparser"
|
"github.com/buger/jsonparser"
|
||||||
log "github.com/cihub/seelog"
|
log "github.com/cihub/seelog"
|
||||||
"infini.sh/console/config"
|
|
||||||
"infini.sh/framework/core/elastic"
|
"infini.sh/framework/core/elastic"
|
||||||
"infini.sh/framework/core/env"
|
"infini.sh/framework/core/env"
|
||||||
|
"infini.sh/framework/core/global"
|
||||||
"infini.sh/framework/core/util"
|
"infini.sh/framework/core/util"
|
||||||
)
|
)
|
||||||
|
|
||||||
func bootstrapRequirementCheck() error{
|
func bootstrapRequirementCheck() error{
|
||||||
err := checkElasticsearchRequire()
|
err := checkElasticsearchRequirements()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -23,7 +23,7 @@ func bootstrapRequirementCheck() error{
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
func checkElasticsearchRequire() error{
|
func checkElasticsearchRequirements() error{
|
||||||
log.Trace("start to check elasticsearch requirement")
|
log.Trace("start to check elasticsearch requirement")
|
||||||
var esConfigs = []elastic.ElasticsearchConfig{}
|
var esConfigs = []elastic.ElasticsearchConfig{}
|
||||||
ok, err := env.ParseConfig("elasticsearch", &esConfigs)
|
ok, err := env.ParseConfig("elasticsearch", &esConfigs)
|
||||||
|
@ -33,27 +33,24 @@ func checkElasticsearchRequire() error{
|
||||||
if !ok {
|
if !ok {
|
||||||
return fmt.Errorf("elasticsearch config section not found")
|
return fmt.Errorf("elasticsearch config section not found")
|
||||||
}
|
}
|
||||||
appConfig = &config.AppConfig{
|
|
||||||
Elasticsearch: "default",
|
elasticsearchID:=global.Lookup(elastic.GlobalSystemElasticsearchID)
|
||||||
}
|
|
||||||
ok, err = env.ParseConfig("web", appConfig)
|
if elasticsearchID == nil||elasticsearchID=="" {
|
||||||
if err != nil {
|
return fmt.Errorf("elasticsearch config in web section can not be empty")
|
||||||
return fmt.Errorf("parse web config section error: %v", err)
|
|
||||||
}
|
|
||||||
if !ok {
|
|
||||||
return fmt.Errorf("web config section not found")
|
|
||||||
}
|
|
||||||
if appConfig.Elasticsearch == "" {
|
|
||||||
return fmt.Errorf("elasticsearch config of web section can not be empty")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
esID:=elasticsearchID.(string)
|
||||||
|
|
||||||
var targetEsConfig *elastic.ElasticsearchConfig
|
var targetEsConfig *elastic.ElasticsearchConfig
|
||||||
for _, esConfig := range esConfigs {
|
for _, esConfig := range esConfigs {
|
||||||
if esConfig.Name == appConfig.Elasticsearch {
|
if esConfig.ID == esID||(esConfig.ID==""&&esConfig.Name==esID) {
|
||||||
targetEsConfig = &esConfig
|
targetEsConfig = &esConfig
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if targetEsConfig == nil {
|
if targetEsConfig == nil {
|
||||||
return fmt.Errorf("elasticsearch config named %s not found", appConfig.Elasticsearch)
|
return fmt.Errorf("elasticsearch config %s was not found", esID)
|
||||||
}
|
}
|
||||||
var req = util.NewGetRequest(targetEsConfig.Endpoint, nil)
|
var req = util.NewGetRequest(targetEsConfig.Endpoint, nil)
|
||||||
if targetEsConfig.BasicAuth != nil {
|
if targetEsConfig.BasicAuth != nil {
|
||||||
|
@ -65,6 +62,10 @@ func checkElasticsearchRequire() error{
|
||||||
return fmt.Errorf("check elasticsearch requirement error: %v", err)
|
return fmt.Errorf("check elasticsearch requirement error: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if result==nil||result.Body==nil||len(result.Body)==0{
|
||||||
|
return fmt.Errorf("failed to retrive elasticsearch version info")
|
||||||
|
}
|
||||||
|
|
||||||
versionNumber, err := jsonparser.GetString(result.Body, "version", "number")
|
versionNumber, err := jsonparser.GetString(result.Body, "version", "number")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("check elasticsearch requirement error: %v, got response: %s", err, string(result.Body))
|
return fmt.Errorf("check elasticsearch requirement error: %v, got response: %s", err, string(result.Body))
|
|
@ -3,7 +3,6 @@ package config
|
||||||
import "infini.sh/framework/core/config"
|
import "infini.sh/framework/core/config"
|
||||||
|
|
||||||
type AppConfig struct {
|
type AppConfig struct {
|
||||||
Elasticsearch string `config:"elasticsearch"`
|
|
||||||
UI UIConfig `config:"ui"`
|
UI UIConfig `config:"ui"`
|
||||||
Network config.NetworkConfig `config:"network"`
|
Network config.NetworkConfig `config:"network"`
|
||||||
TLSConfig config.TLSConfig `config:"tls"`
|
TLSConfig config.TLSConfig `config:"tls"`
|
||||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,84 @@
|
||||||
|
|
||||||
|
elasticsearch:
|
||||||
|
- id: $[[CLUSTER_ID]]
|
||||||
|
name: $[[CLUSTER_ID]]
|
||||||
|
enabled: true
|
||||||
|
monitored: true
|
||||||
|
reserved: true
|
||||||
|
endpoint: $[[CLUSTER_ENDPINT]]
|
||||||
|
basic_auth:
|
||||||
|
username: $[[CLUSTER_USER]]
|
||||||
|
password: $[[CLUSTER_PASS]]
|
||||||
|
|
||||||
|
elastic.elasticsearch: $[[CLUSTER_ID]]
|
||||||
|
|
||||||
|
pipeline:
|
||||||
|
- name: indexing_merge
|
||||||
|
auto_start: true
|
||||||
|
keep_running: true
|
||||||
|
processor:
|
||||||
|
- indexing_merge:
|
||||||
|
input_queue: "metrics"
|
||||||
|
elasticsearch: "$[[CLUSTER_ID]]"
|
||||||
|
index_name: "$[[INDEX_PREFIX]]metrics"
|
||||||
|
output_queue:
|
||||||
|
name: "metrics_requests"
|
||||||
|
label:
|
||||||
|
tag: "metrics"
|
||||||
|
worker_size: 1
|
||||||
|
bulk_size_in_mb: 5
|
||||||
|
- name: consume-metrics_requests
|
||||||
|
auto_start: true
|
||||||
|
keep_running: true
|
||||||
|
processor:
|
||||||
|
- bulk_indexing:
|
||||||
|
bulk:
|
||||||
|
compress: true
|
||||||
|
batch_size_in_mb: 5
|
||||||
|
batch_size_in_docs: 5000
|
||||||
|
consumer:
|
||||||
|
fetch_max_messages: 100
|
||||||
|
queues:
|
||||||
|
type: indexing_merge
|
||||||
|
when:
|
||||||
|
cluster_available: ["$[[CLUSTER_ID]]"]
|
||||||
|
- name: metadata_ingest
|
||||||
|
auto_start: true
|
||||||
|
keep_running: true
|
||||||
|
processor:
|
||||||
|
- metadata:
|
||||||
|
bulk_size_in_mb: 5
|
||||||
|
bulk_max_docs_count: 5000
|
||||||
|
fetch_max_messages: 100
|
||||||
|
elasticsearch: "$[[CLUSTER_ID]]"
|
||||||
|
queues:
|
||||||
|
type: metadata
|
||||||
|
category: elasticsearch
|
||||||
|
consumer:
|
||||||
|
group: metadata
|
||||||
|
when:
|
||||||
|
cluster_available: ["$[[CLUSTER_ID]]"]
|
||||||
|
- name: activity_ingest
|
||||||
|
auto_start: true
|
||||||
|
keep_running: true
|
||||||
|
processor:
|
||||||
|
- activity:
|
||||||
|
bulk_size_in_mb: 5
|
||||||
|
bulk_max_docs_count: 5000
|
||||||
|
fetch_max_messages: 100
|
||||||
|
elasticsearch: "$[[CLUSTER_ID]]"
|
||||||
|
queues:
|
||||||
|
category: elasticsearch
|
||||||
|
activity: true
|
||||||
|
consumer:
|
||||||
|
group: activity
|
||||||
|
when:
|
||||||
|
cluster_available: ["$[[CLUSTER_ID]]"]
|
||||||
|
- name: cluster_migration_split
|
||||||
|
auto_start: true
|
||||||
|
keep_running: true
|
||||||
|
processor:
|
||||||
|
- cluster_migration:
|
||||||
|
elasticsearch: "$[[CLUSTER_ID]]"
|
||||||
|
when:
|
||||||
|
cluster_available: ["$[[CLUSTER_ID]]"]
|
80
console.yml
80
console.yml
|
@ -1,20 +1,10 @@
|
||||||
# for the system cluster, please use Elasticsearch v7.3+
|
path.configs: "config"
|
||||||
elasticsearch:
|
|
||||||
- name: default
|
|
||||||
enabled: true
|
|
||||||
monitored: false
|
|
||||||
endpoint: http://localhost:9200
|
|
||||||
basic_auth:
|
|
||||||
username: elastic
|
|
||||||
password: infinilabs
|
|
||||||
discovery:
|
|
||||||
enabled: true
|
|
||||||
|
|
||||||
web:
|
web:
|
||||||
enabled: true
|
enabled: true
|
||||||
embedding_api: true
|
embedding_api: true
|
||||||
auth:
|
auth:
|
||||||
enabled: false
|
enabled: true
|
||||||
ui:
|
ui:
|
||||||
enabled: true
|
enabled: true
|
||||||
path: .public
|
path: .public
|
||||||
|
@ -27,7 +17,6 @@ web:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
|
||||||
elastic:
|
elastic:
|
||||||
elasticsearch: default
|
|
||||||
enabled: true
|
enabled: true
|
||||||
remote_configs: true
|
remote_configs: true
|
||||||
health_check:
|
health_check:
|
||||||
|
@ -58,67 +47,4 @@ metrics:
|
||||||
enabled: true
|
enabled: true
|
||||||
cluster_stats: true
|
cluster_stats: true
|
||||||
node_stats: true
|
node_stats: true
|
||||||
index_stats: true
|
index_stats: true
|
||||||
|
|
||||||
pipeline:
|
|
||||||
- name: indexing_merge
|
|
||||||
auto_start: true
|
|
||||||
keep_running: true
|
|
||||||
processor:
|
|
||||||
- indexing_merge:
|
|
||||||
input_queue: "metrics"
|
|
||||||
elasticsearch: "default"
|
|
||||||
index_name: ".infini_metrics"
|
|
||||||
output_queue:
|
|
||||||
name: "metrics_requests"
|
|
||||||
label:
|
|
||||||
tag: "metrics"
|
|
||||||
worker_size: 1
|
|
||||||
bulk_size_in_mb: 10
|
|
||||||
- name: consume-metrics_requests
|
|
||||||
auto_start: true
|
|
||||||
keep_running: true
|
|
||||||
processor:
|
|
||||||
- bulk_indexing:
|
|
||||||
bulk:
|
|
||||||
compress: true
|
|
||||||
batch_size_in_mb: 10
|
|
||||||
batch_size_in_docs: 5000
|
|
||||||
consumer:
|
|
||||||
fetch_max_messages: 100
|
|
||||||
queues:
|
|
||||||
type: indexing_merge
|
|
||||||
when:
|
|
||||||
cluster_available: ["default"]
|
|
||||||
- name: metadata_ingest
|
|
||||||
auto_start: true
|
|
||||||
keep_running: true
|
|
||||||
processor:
|
|
||||||
- metadata:
|
|
||||||
bulk_size_in_mb: 10
|
|
||||||
bulk_max_docs_count: 5000
|
|
||||||
fetch_max_messages: 1000
|
|
||||||
elasticsearch: "default"
|
|
||||||
queues:
|
|
||||||
type: metadata
|
|
||||||
category: elasticsearch
|
|
||||||
consumer:
|
|
||||||
group: metadata
|
|
||||||
when:
|
|
||||||
cluster_available: ["default"]
|
|
||||||
- name: activity_ingest
|
|
||||||
auto_start: true
|
|
||||||
keep_running: true
|
|
||||||
processor:
|
|
||||||
- activity:
|
|
||||||
bulk_size_in_mb: 10
|
|
||||||
bulk_max_docs_count: 5000
|
|
||||||
fetch_max_messages: 1000
|
|
||||||
elasticsearch: "default"
|
|
||||||
queues:
|
|
||||||
category: elasticsearch
|
|
||||||
activity: true
|
|
||||||
consumer:
|
|
||||||
group: activity
|
|
||||||
when:
|
|
||||||
cluster_available: ["default"]
|
|
133
main.go
133
main.go
|
@ -1,6 +1,7 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
_ "expvar"
|
_ "expvar"
|
||||||
log "github.com/cihub/seelog"
|
log "github.com/cihub/seelog"
|
||||||
|
@ -12,11 +13,12 @@ import (
|
||||||
"infini.sh/framework"
|
"infini.sh/framework"
|
||||||
"infini.sh/framework/core/elastic"
|
"infini.sh/framework/core/elastic"
|
||||||
"infini.sh/framework/core/env"
|
"infini.sh/framework/core/env"
|
||||||
|
"infini.sh/framework/core/global"
|
||||||
"infini.sh/framework/core/insight"
|
"infini.sh/framework/core/insight"
|
||||||
_ "infini.sh/framework/core/log"
|
_ "infini.sh/framework/core/log"
|
||||||
"infini.sh/framework/core/module"
|
"infini.sh/framework/core/module"
|
||||||
"infini.sh/framework/core/orm"
|
"infini.sh/framework/core/orm"
|
||||||
task2 "infini.sh/framework/core/task"
|
task1 "infini.sh/framework/core/task"
|
||||||
"infini.sh/framework/modules/agent"
|
"infini.sh/framework/modules/agent"
|
||||||
_ "infini.sh/framework/modules/api"
|
_ "infini.sh/framework/modules/api"
|
||||||
elastic2 "infini.sh/framework/modules/elastic"
|
elastic2 "infini.sh/framework/modules/elastic"
|
||||||
|
@ -25,10 +27,13 @@ import (
|
||||||
"infini.sh/framework/modules/pipeline"
|
"infini.sh/framework/modules/pipeline"
|
||||||
queue2 "infini.sh/framework/modules/queue/disk_queue"
|
queue2 "infini.sh/framework/modules/queue/disk_queue"
|
||||||
"infini.sh/framework/modules/redis"
|
"infini.sh/framework/modules/redis"
|
||||||
|
"infini.sh/framework/modules/security"
|
||||||
"infini.sh/framework/modules/stats"
|
"infini.sh/framework/modules/stats"
|
||||||
"infini.sh/framework/modules/task"
|
"infini.sh/framework/modules/task"
|
||||||
"infini.sh/framework/modules/ui"
|
"infini.sh/framework/modules/ui"
|
||||||
_ "infini.sh/framework/plugins"
|
_ "infini.sh/framework/plugins"
|
||||||
|
setup1 "infini.sh/console/plugin/setup"
|
||||||
|
_ "infini.sh/console/plugin"
|
||||||
api2 "infini.sh/gateway/api"
|
api2 "infini.sh/gateway/api"
|
||||||
_ "infini.sh/gateway/proxy"
|
_ "infini.sh/gateway/proxy"
|
||||||
_ "time/tzdata"
|
_ "time/tzdata"
|
||||||
|
@ -38,12 +43,7 @@ var appConfig *config.AppConfig
|
||||||
var appUI *UI
|
var appUI *UI
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
terminalHeader := ("\n\n")
|
terminalHeader := ("\n")
|
||||||
terminalHeader += (" ___ __ ___ ___ \n")
|
|
||||||
terminalHeader += (" / __\\/ / /___\\/\\ /\\ / \\ \n")
|
|
||||||
terminalHeader += (" / / / / // // / \\ \\/ /\\ / \n")
|
|
||||||
terminalHeader += ("/ /__/ /__/ \\_//\\ \\_/ / /_// \n")
|
|
||||||
terminalHeader += ("\\____|____|___/ \\___/___,' \n")
|
|
||||||
terminalHeader += (" ___ ___ __ __ ___ __ __ \n")
|
terminalHeader += (" ___ ___ __ __ ___ __ __ \n")
|
||||||
terminalHeader += (" / __\\/___\\/\\ \\ \\/ _\\ /___\\/ / /__\\\n")
|
terminalHeader += (" / __\\/___\\/\\ \\ \\/ _\\ /___\\/ / /__\\\n")
|
||||||
terminalHeader += (" / / // // \\/ /\\ \\ // // / /_\\ \n")
|
terminalHeader += (" / / // // \\/ /\\ \\ // // / /_\\ \n")
|
||||||
|
@ -53,7 +53,7 @@ func main() {
|
||||||
|
|
||||||
terminalFooter := ""
|
terminalFooter := ""
|
||||||
|
|
||||||
app := framework.NewApp("console", "INFINI Cloud Console, The easiest way to operate your own elasticsearch platform.",
|
app := framework.NewApp("console", "The easiest way to operate your own search platform.",
|
||||||
config.Version, config.BuildNumber, config.LastCommitLog, config.BuildDate, config.EOLDate, terminalHeader, terminalFooter)
|
config.Version, config.BuildNumber, config.LastCommitLog, config.BuildDate, config.EOLDate, terminalHeader, terminalFooter)
|
||||||
|
|
||||||
app.Init(nil)
|
app.Init(nil)
|
||||||
|
@ -61,29 +61,51 @@ func main() {
|
||||||
|
|
||||||
api := api2.GatewayAPI{}
|
api := api2.GatewayAPI{}
|
||||||
|
|
||||||
|
modules:=[]module.Module{}
|
||||||
|
modules=append(modules,&stats.SimpleStatsModule{})
|
||||||
|
modules=append(modules,&elastic2.ElasticModule{})
|
||||||
|
modules=append(modules,&queue2.DiskQueue{})
|
||||||
|
modules=append(modules,&redis.RedisModule{})
|
||||||
|
modules=append(modules,&pipeline.PipeModule{})
|
||||||
|
modules=append(modules,&task.TaskModule{})
|
||||||
|
modules=append(modules,&agent.AgentModule{})
|
||||||
|
modules=append(modules,&metrics.MetricsModule{})
|
||||||
|
modules=append(modules,&security.Module{})
|
||||||
|
modules=append(modules,&migration.MigrationModule{})
|
||||||
|
|
||||||
|
uiModule:=&ui.UIModule{}
|
||||||
|
|
||||||
if app.Setup(func() {
|
if app.Setup(func() {
|
||||||
err := bootstrapRequirementCheck()
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
//load core modules first
|
//load core modules first
|
||||||
module.RegisterSystemModule(&elastic2.ElasticModule{})
|
module.RegisterSystemModule(&setup1.Module{})
|
||||||
module.RegisterSystemModule(&stats.SimpleStatsModule{})
|
module.RegisterSystemModule(uiModule)
|
||||||
module.RegisterSystemModule(&queue2.DiskQueue{})
|
|
||||||
module.RegisterSystemModule(&redis.RedisModule{})
|
|
||||||
module.RegisterSystemModule(&ui.UIModule{})
|
|
||||||
module.RegisterSystemModule(&pipeline.PipeModule{})
|
|
||||||
module.RegisterSystemModule(&task.TaskModule{})
|
|
||||||
module.RegisterSystemModule(&agent.AgentModule{})
|
|
||||||
module.RegisterSystemModule(&migration.MigrationModule{})
|
|
||||||
|
|
||||||
module.RegisterUserPlugin(&metrics.MetricsModule{})
|
var initFunc= func() {
|
||||||
|
module.RegisterSystemModule(&stats.SimpleStatsModule{})
|
||||||
|
module.RegisterSystemModule(&elastic2.ElasticModule{})
|
||||||
|
module.RegisterSystemModule(&queue2.DiskQueue{})
|
||||||
|
module.RegisterSystemModule(&redis.RedisModule{})
|
||||||
|
module.RegisterSystemModule(&pipeline.PipeModule{})
|
||||||
|
module.RegisterSystemModule(&task.TaskModule{})
|
||||||
|
module.RegisterSystemModule(&agent.AgentModule{})
|
||||||
|
module.RegisterSystemModule(&metrics.MetricsModule{})
|
||||||
|
module.RegisterSystemModule(&security.Module{})
|
||||||
|
module.RegisterSystemModule(&migration.MigrationModule{})
|
||||||
|
}
|
||||||
|
|
||||||
|
if !global.Env().SetupRequired(){
|
||||||
|
initFunc()
|
||||||
|
}else{
|
||||||
|
for _, v := range modules {
|
||||||
|
v.Setup()
|
||||||
|
}
|
||||||
|
setup1.RegisterSetupCallback(initFunc)
|
||||||
|
}
|
||||||
|
|
||||||
api.RegisterAPI("")
|
api.RegisterAPI("")
|
||||||
|
|
||||||
appConfig = &config.AppConfig{
|
appConfig = &config.AppConfig{
|
||||||
Elasticsearch: "default",
|
|
||||||
UI: config.UIConfig{
|
UI: config.UIConfig{
|
||||||
LocalPath: ".public",
|
LocalPath: ".public",
|
||||||
VFSEnabled: true,
|
VFSEnabled: true,
|
||||||
|
@ -107,28 +129,51 @@ func main() {
|
||||||
|
|
||||||
module.Start()
|
module.Start()
|
||||||
|
|
||||||
//orm.RegisterSchemaWithIndexName(model.Dict{}, "dict")
|
var initFunc= func() {
|
||||||
//orm.RegisterSchemaWithIndexName(model.Reindex{}, "reindex")
|
if global.Env().SetupRequired() {
|
||||||
orm.RegisterSchemaWithIndexName(elastic.View{}, "view")
|
for _, v := range modules {
|
||||||
orm.RegisterSchemaWithIndexName(elastic.CommonCommand{}, "commands")
|
v.Start()
|
||||||
//orm.RegisterSchemaWithIndexName(elastic.TraceTemplate{}, "trace-template")
|
}
|
||||||
orm.RegisterSchemaWithIndexName(gateway.Instance{}, "gateway-instance")
|
|
||||||
orm.RegisterSchemaWithIndexName(alerting.Rule{}, "alert-rule")
|
|
||||||
orm.RegisterSchemaWithIndexName(alerting.Alert{}, "alert-history")
|
|
||||||
orm.RegisterSchemaWithIndexName(alerting.AlertMessage{}, "alert-message")
|
|
||||||
orm.RegisterSchemaWithIndexName(alerting.Channel{}, "channel")
|
|
||||||
orm.RegisterSchemaWithIndexName(insight.Visualization{}, "visualization")
|
|
||||||
orm.RegisterSchemaWithIndexName(insight.Dashboard{}, "dashboard")
|
|
||||||
orm.RegisterSchemaWithIndexName(task2.Task{}, "task")
|
|
||||||
orm.RegisterSchemaWithIndexName(task2.Log{}, "task-log")
|
|
||||||
api.RegisterSchema()
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
err := alerting2.InitTasks()
|
|
||||||
if err != nil {
|
|
||||||
log.Errorf("init alerting task error: %v", err)
|
|
||||||
}
|
}
|
||||||
}()
|
|
||||||
|
elastic2.InitTemplate(false)
|
||||||
|
|
||||||
|
//orm.RegisterSchemaWithIndexName(model.Dict{}, "dict")
|
||||||
|
orm.RegisterSchemaWithIndexName(elastic.View{}, "view")
|
||||||
|
orm.RegisterSchemaWithIndexName(elastic.CommonCommand{}, "commands")
|
||||||
|
//orm.RegisterSchemaWithIndexName(elastic.TraceTemplate{}, "trace-template")
|
||||||
|
orm.RegisterSchemaWithIndexName(gateway.Instance{}, "gateway-instance")
|
||||||
|
orm.RegisterSchemaWithIndexName(alerting.Rule{}, "alert-rule")
|
||||||
|
orm.RegisterSchemaWithIndexName(alerting.Alert{}, "alert-history")
|
||||||
|
orm.RegisterSchemaWithIndexName(alerting.AlertMessage{}, "alert-message")
|
||||||
|
orm.RegisterSchemaWithIndexName(alerting.Channel{}, "channel")
|
||||||
|
orm.RegisterSchemaWithIndexName(insight.Visualization{}, "visualization")
|
||||||
|
orm.RegisterSchemaWithIndexName(insight.Dashboard{}, "dashboard")
|
||||||
|
orm.RegisterSchemaWithIndexName(task1.Task{}, "task")
|
||||||
|
orm.RegisterSchemaWithIndexName(task1.Log{}, "task-log")
|
||||||
|
api.RegisterSchema()
|
||||||
|
|
||||||
|
task1.RunWithinGroup("initialize_alerting",func(ctx context.Context) error {
|
||||||
|
err := alerting2.InitTasks()
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("init alerting task error: %v", err)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if !global.Env().SetupRequired(){
|
||||||
|
initFunc()
|
||||||
|
}else{
|
||||||
|
setup1.RegisterSetupCallback(initFunc)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !global.Env().SetupRequired(){
|
||||||
|
err := bootstrapRequirementCheck()
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
}, nil) {
|
}, nil) {
|
||||||
app.Run()
|
app.Run()
|
||||||
|
|
|
@ -10,6 +10,7 @@ import (
|
||||||
"infini.sh/console/model/alerting"
|
"infini.sh/console/model/alerting"
|
||||||
httprouter "infini.sh/framework/core/api/router"
|
httprouter "infini.sh/framework/core/api/router"
|
||||||
"infini.sh/framework/core/elastic"
|
"infini.sh/framework/core/elastic"
|
||||||
|
"infini.sh/framework/core/global"
|
||||||
"infini.sh/framework/core/orm"
|
"infini.sh/framework/core/orm"
|
||||||
"infini.sh/framework/core/util"
|
"infini.sh/framework/core/util"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
@ -109,7 +110,7 @@ func (h *AlertAPI) searchAlert(w http.ResponseWriter, req *http.Request, ps http
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *AlertAPI) getAlertStats(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
|
func (h *AlertAPI) getAlertStats(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
|
||||||
esClient := elastic.GetClient(h.Config.Elasticsearch)
|
esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
|
||||||
queryDsl := util.MapStr{
|
queryDsl := util.MapStr{
|
||||||
"size": 0,
|
"size": 0,
|
||||||
"query": util.MapStr{
|
"query": util.MapStr{
|
||||||
|
|
|
@ -5,7 +5,6 @@
|
||||||
package alerting
|
package alerting
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"infini.sh/console/config"
|
|
||||||
"infini.sh/framework/core/api"
|
"infini.sh/framework/core/api"
|
||||||
"infini.sh/framework/core/api/rbac/enum"
|
"infini.sh/framework/core/api/rbac/enum"
|
||||||
)
|
)
|
||||||
|
@ -13,7 +12,6 @@ import (
|
||||||
|
|
||||||
type AlertAPI struct {
|
type AlertAPI struct {
|
||||||
api.Handler
|
api.Handler
|
||||||
Config *config.AppConfig
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (alert *AlertAPI) Init() {
|
func (alert *AlertAPI) Init() {
|
||||||
|
|
|
@ -11,6 +11,7 @@ import (
|
||||||
alerting2 "infini.sh/console/service/alerting"
|
alerting2 "infini.sh/console/service/alerting"
|
||||||
httprouter "infini.sh/framework/core/api/router"
|
httprouter "infini.sh/framework/core/api/router"
|
||||||
"infini.sh/framework/core/elastic"
|
"infini.sh/framework/core/elastic"
|
||||||
|
"infini.sh/framework/core/global"
|
||||||
"infini.sh/framework/core/kv"
|
"infini.sh/framework/core/kv"
|
||||||
"infini.sh/framework/core/orm"
|
"infini.sh/framework/core/orm"
|
||||||
"infini.sh/framework/core/util"
|
"infini.sh/framework/core/util"
|
||||||
|
@ -82,7 +83,7 @@ func (h *AlertAPI) ignoreAlertMessage(w http.ResponseWriter, req *http.Request,
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *AlertAPI) getAlertMessageStats(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
|
func (h *AlertAPI) getAlertMessageStats(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
|
||||||
esClient := elastic.GetClient(h.Config.Elasticsearch)
|
esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
|
||||||
must := []util.MapStr{
|
must := []util.MapStr{
|
||||||
{
|
{
|
||||||
"terms": util.MapStr{
|
"terms": util.MapStr{
|
||||||
|
|
|
@ -14,6 +14,7 @@ import (
|
||||||
httprouter "infini.sh/framework/core/api/router"
|
httprouter "infini.sh/framework/core/api/router"
|
||||||
"infini.sh/framework/core/elastic"
|
"infini.sh/framework/core/elastic"
|
||||||
"infini.sh/framework/core/event"
|
"infini.sh/framework/core/event"
|
||||||
|
"infini.sh/framework/core/global"
|
||||||
"infini.sh/framework/core/kv"
|
"infini.sh/framework/core/kv"
|
||||||
"infini.sh/framework/core/orm"
|
"infini.sh/framework/core/orm"
|
||||||
"infini.sh/framework/core/queue"
|
"infini.sh/framework/core/queue"
|
||||||
|
@ -462,7 +463,8 @@ func (alertAPI *AlertAPI) searchRule(w http.ResponseWriter, req *http.Request, p
|
||||||
}
|
}
|
||||||
|
|
||||||
func (alertAPI *AlertAPI) getRuleAlertMessageNumbers(ruleIDs []string) ( map[string]interface{},error) {
|
func (alertAPI *AlertAPI) getRuleAlertMessageNumbers(ruleIDs []string) ( map[string]interface{},error) {
|
||||||
esClient := elastic.GetClient(alertAPI.Config.Elasticsearch)
|
|
||||||
|
esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
|
||||||
queryDsl := util.MapStr{
|
queryDsl := util.MapStr{
|
||||||
"size": 0,
|
"size": 0,
|
||||||
"query": util.MapStr{
|
"query": util.MapStr{
|
||||||
|
@ -513,7 +515,7 @@ func (alertAPI *AlertAPI) fetchAlertInfos(w http.ResponseWriter, req *http.Reque
|
||||||
alertAPI.WriteJSON(w, util.MapStr{}, http.StatusOK)
|
alertAPI.WriteJSON(w, util.MapStr{}, http.StatusOK)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
esClient := elastic.GetClient(alertAPI.Config.Elasticsearch)
|
esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
|
||||||
queryDsl := util.MapStr{
|
queryDsl := util.MapStr{
|
||||||
"_source": []string{"state", "rule_id"},
|
"_source": []string{"state", "rule_id"},
|
||||||
"sort": []util.MapStr{
|
"sort": []util.MapStr{
|
||||||
|
|
|
@ -13,7 +13,7 @@ type GatewayAPI struct {
|
||||||
api.Handler
|
api.Handler
|
||||||
}
|
}
|
||||||
|
|
||||||
func init() {
|
func InitAPI() {
|
||||||
gateway:=GatewayAPI{}
|
gateway:=GatewayAPI{}
|
||||||
api.HandleAPIMethod(api.POST, "/gateway/instance/try_connect", gateway.RequireLogin(gateway.tryConnect))
|
api.HandleAPIMethod(api.POST, "/gateway/instance/try_connect", gateway.RequireLogin(gateway.tryConnect))
|
||||||
api.HandleAPIMethod(api.GET, "/gateway/instance/:instance_id", gateway.RequirePermission(gateway.getInstance, enum.PermissionGatewayInstanceRead))
|
api.HandleAPIMethod(api.GET, "/gateway/instance/:instance_id", gateway.RequirePermission(gateway.getInstance, enum.PermissionGatewayInstanceRead))
|
||||||
|
|
|
@ -5,6 +5,7 @@ import (
|
||||||
log "github.com/cihub/seelog"
|
log "github.com/cihub/seelog"
|
||||||
httprouter "infini.sh/framework/core/api/router"
|
httprouter "infini.sh/framework/core/api/router"
|
||||||
"infini.sh/framework/core/elastic"
|
"infini.sh/framework/core/elastic"
|
||||||
|
"infini.sh/framework/core/global"
|
||||||
"infini.sh/framework/core/orm"
|
"infini.sh/framework/core/orm"
|
||||||
"infini.sh/framework/core/util"
|
"infini.sh/framework/core/util"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
@ -28,7 +29,7 @@ func (h *APIHandler) HandleAddCommonCommandAction(w http.ResponseWriter, req *ht
|
||||||
|
|
||||||
reqParams.Created = time.Now()
|
reqParams.Created = time.Now()
|
||||||
reqParams.ID = util.GetUUID()
|
reqParams.ID = util.GetUUID()
|
||||||
esClient := elastic.GetClient(h.Config.Elasticsearch)
|
esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
|
||||||
|
|
||||||
queryDSL :=[]byte(fmt.Sprintf(`{"size":1, "query":{"bool":{"must":{"match":{"title.keyword":"%s"}}}}}`, reqParams.Title))
|
queryDSL :=[]byte(fmt.Sprintf(`{"size":1, "query":{"bool":{"must":{"match":{"title.keyword":"%s"}}}}}`, reqParams.Title))
|
||||||
var indexName = orm.GetIndexName(reqParams)
|
var indexName = orm.GetIndexName(reqParams)
|
||||||
|
@ -73,7 +74,7 @@ func (h *APIHandler) HandleSaveCommonCommandAction(w http.ResponseWriter, req *h
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
reqParams.ID = ps.ByName("cid")
|
reqParams.ID = ps.ByName("cid")
|
||||||
esClient := elastic.GetClient(h.Config.Elasticsearch)
|
esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
|
||||||
|
|
||||||
queryDSL :=[]byte(fmt.Sprintf(`{"size":1, "query":{"bool":{"must":{"match":{"title.keyword":"%s"}}}}}`, reqParams.Title))
|
queryDSL :=[]byte(fmt.Sprintf(`{"size":1, "query":{"bool":{"must":{"match":{"title.keyword":"%s"}}}}}`, reqParams.Title))
|
||||||
var indexName = orm.GetIndexName(reqParams)
|
var indexName = orm.GetIndexName(reqParams)
|
||||||
|
@ -133,7 +134,7 @@ func (h *APIHandler) HandleQueryCommonCommandAction(w http.ResponseWriter, req *
|
||||||
}
|
}
|
||||||
|
|
||||||
queryDSL = fmt.Sprintf(queryDSL, filterBuilder.String(), size, from)
|
queryDSL = fmt.Sprintf(queryDSL, filterBuilder.String(), size, from)
|
||||||
esClient := elastic.GetClient(h.Config.Elasticsearch)
|
esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
|
||||||
|
|
||||||
searchRes, err := esClient.SearchWithRawQueryDSL(orm.GetIndexName(elastic.CommonCommand{}), []byte(queryDSL))
|
searchRes, err := esClient.SearchWithRawQueryDSL(orm.GetIndexName(elastic.CommonCommand{}), []byte(queryDSL))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -149,7 +150,7 @@ func (h *APIHandler) HandleQueryCommonCommandAction(w http.ResponseWriter, req *
|
||||||
func (h *APIHandler) HandleDeleteCommonCommandAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
|
func (h *APIHandler) HandleDeleteCommonCommandAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
|
||||||
resBody := map[string]interface{}{}
|
resBody := map[string]interface{}{}
|
||||||
id := ps.ByName("cid")
|
id := ps.ByName("cid")
|
||||||
esClient := elastic.GetClient(h.Config.Elasticsearch)
|
esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
|
||||||
delRes, err := esClient.Delete(orm.GetIndexName(elastic.CommonCommand{}), "", id, "wait_for")
|
delRes, err := esClient.Delete(orm.GetIndexName(elastic.CommonCommand{}), "", id, "wait_for")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error(err)
|
log.Error(err)
|
||||||
|
|
|
@ -6,6 +6,7 @@ import (
|
||||||
httprouter "infini.sh/framework/core/api/router"
|
httprouter "infini.sh/framework/core/api/router"
|
||||||
"infini.sh/framework/core/elastic"
|
"infini.sh/framework/core/elastic"
|
||||||
"infini.sh/framework/core/event"
|
"infini.sh/framework/core/event"
|
||||||
|
"infini.sh/framework/core/global"
|
||||||
"infini.sh/framework/core/host"
|
"infini.sh/framework/core/host"
|
||||||
"infini.sh/framework/core/orm"
|
"infini.sh/framework/core/orm"
|
||||||
"infini.sh/framework/core/util"
|
"infini.sh/framework/core/util"
|
||||||
|
@ -25,7 +26,7 @@ func (handler APIHandler) ElasticsearchOverviewAction(w http.ResponseWriter, req
|
||||||
// clusterIDs = append(clusterIDs, key)
|
// clusterIDs = append(clusterIDs, key)
|
||||||
// return true
|
// return true
|
||||||
//})
|
//})
|
||||||
esClient := elastic.GetClient(handler.Config.Elasticsearch)
|
esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
|
||||||
queryDsl := util.MapStr{
|
queryDsl := util.MapStr{
|
||||||
"size": 100,
|
"size": 100,
|
||||||
}
|
}
|
||||||
|
@ -110,7 +111,7 @@ func (handler APIHandler) ElasticsearchOverviewAction(w http.ResponseWriter, req
|
||||||
}
|
}
|
||||||
|
|
||||||
func (handler APIHandler) getLatestClusterMonitorData(clusterIDs []interface{}) (*elastic.SearchResponse, error){
|
func (handler APIHandler) getLatestClusterMonitorData(clusterIDs []interface{}) (*elastic.SearchResponse, error){
|
||||||
client := elastic.GetClient(handler.Config.Elasticsearch)
|
client := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
|
||||||
queryDSLTpl := `{
|
queryDSLTpl := `{
|
||||||
"size": %d,
|
"size": %d,
|
||||||
"query": {
|
"query": {
|
||||||
|
@ -155,7 +156,7 @@ func (handler APIHandler) getLatestClusterMonitorData(clusterIDs []interface{})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (handler APIHandler) getMetricCount(indexName, field string, clusterIDs []interface{}) (interface{}, error){
|
func (handler APIHandler) getMetricCount(indexName, field string, clusterIDs []interface{}) (interface{}, error){
|
||||||
client := elastic.GetClient(handler.Config.Elasticsearch)
|
client := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
|
||||||
queryDSL := util.MapStr{
|
queryDSL := util.MapStr{
|
||||||
"size": 0,
|
"size": 0,
|
||||||
"aggs": util.MapStr{
|
"aggs": util.MapStr{
|
||||||
|
@ -182,7 +183,7 @@ func (handler APIHandler) getMetricCount(indexName, field string, clusterIDs []i
|
||||||
}
|
}
|
||||||
|
|
||||||
func (handler APIHandler) getLastActiveHostCount() (int, error){
|
func (handler APIHandler) getLastActiveHostCount() (int, error){
|
||||||
client := elastic.GetClient(handler.Config.Elasticsearch)
|
client := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
|
||||||
queryDSL := `{
|
queryDSL := `{
|
||||||
"size": 0,
|
"size": 0,
|
||||||
"query": {
|
"query": {
|
||||||
|
|
|
@ -1,6 +1,8 @@
|
||||||
package index_management
|
package index_management
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"infini.sh/framework/core/elastic"
|
||||||
|
"infini.sh/framework/core/global"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
@ -33,7 +35,7 @@ func (handler APIHandler) GetDictListAction(w http.ResponseWriter, req *http.Req
|
||||||
if len(tags) > 3 {
|
if len(tags) > 3 {
|
||||||
tags = tags[0:3]
|
tags = tags[0:3]
|
||||||
}
|
}
|
||||||
rel, err := model2.GetDictList(from, size, name, tags, handler.Config.Elasticsearch)
|
rel, err := model2.GetDictList(from, size, name, tags, global.MustLookupString(elastic.GlobalSystemElasticsearchID))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
resp["error"] = err
|
resp["error"] = err
|
||||||
resp["status"] = false
|
resp["status"] = false
|
||||||
|
|
|
@ -4,6 +4,7 @@ import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
log "github.com/cihub/seelog"
|
log "github.com/cihub/seelog"
|
||||||
|
"infini.sh/framework/core/global"
|
||||||
"infini.sh/framework/core/orm"
|
"infini.sh/framework/core/orm"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
@ -33,7 +34,7 @@ func (handler APIHandler) HandleReindexAction(w http.ResponseWriter, req *http.R
|
||||||
|
|
||||||
//fmt.Println(reindexItem)
|
//fmt.Println(reindexItem)
|
||||||
typ := handler.GetParameter(req, "_type")
|
typ := handler.GetParameter(req, "_type")
|
||||||
ID, err := reindex(handler.Config.Elasticsearch, reindexItem, typ)
|
ID, err := reindex(global.MustLookupString(elastic.GlobalSystemElasticsearchID), reindexItem, typ)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error(err)
|
log.Error(err)
|
||||||
resResult["error"] = err
|
resResult["error"] = err
|
||||||
|
@ -94,7 +95,7 @@ func (handler APIHandler) HandleDeleteRebuildAction(w http.ResponseWriter, req *
|
||||||
id := ps.ByName("id")
|
id := ps.ByName("id")
|
||||||
var ids = []string{id}
|
var ids = []string{id}
|
||||||
resBody := newResponseBody()
|
resBody := newResponseBody()
|
||||||
err := deleteTasksByIds(handler.Config.Elasticsearch, ids)
|
err := deleteTasksByIds(global.MustLookupString(elastic.GlobalSystemElasticsearchID), ids)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error(err)
|
log.Error(err)
|
||||||
resBody["error"] = err
|
resBody["error"] = err
|
||||||
|
@ -111,7 +112,7 @@ func (handler APIHandler) HandleGetRebuildListAction(w http.ResponseWriter, req
|
||||||
size = handler.GetIntOrDefault(req, "size", 10)
|
size = handler.GetIntOrDefault(req, "size", 10)
|
||||||
name = handler.GetParameter(req, "name")
|
name = handler.GetParameter(req, "name")
|
||||||
resBody = newResponseBody()
|
resBody = newResponseBody()
|
||||||
esName = handler.Config.Elasticsearch
|
esName = global.MustLookupString(elastic.GlobalSystemElasticsearchID)
|
||||||
)
|
)
|
||||||
esResp, err := model.GetRebuildList(esName, from, size, name)
|
esResp, err := model.GetRebuildList(esName, from, size, name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -3,7 +3,9 @@ package api
|
||||||
import (
|
import (
|
||||||
"infini.sh/console/config"
|
"infini.sh/console/config"
|
||||||
"infini.sh/console/plugin/api/alerting"
|
"infini.sh/console/plugin/api/alerting"
|
||||||
|
"infini.sh/console/plugin/api/gateway"
|
||||||
"infini.sh/console/plugin/api/index_management"
|
"infini.sh/console/plugin/api/index_management"
|
||||||
|
"infini.sh/console/plugin/api/insight"
|
||||||
"infini.sh/framework/core/api"
|
"infini.sh/framework/core/api"
|
||||||
"infini.sh/framework/core/api/rbac/enum"
|
"infini.sh/framework/core/api/rbac/enum"
|
||||||
"path"
|
"path"
|
||||||
|
@ -58,9 +60,10 @@ func Init(cfg *config.AppConfig) {
|
||||||
//})
|
//})
|
||||||
|
|
||||||
alertAPI := alerting.AlertAPI{
|
alertAPI := alerting.AlertAPI{
|
||||||
Config: cfg,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
alertAPI.Init()
|
alertAPI.Init()
|
||||||
|
|
||||||
|
gateway.InitAPI()
|
||||||
|
insight.InitAPI()
|
||||||
}
|
}
|
||||||
|
|
|
@ -10,7 +10,7 @@ type InsightAPI struct {
|
||||||
api.Handler
|
api.Handler
|
||||||
}
|
}
|
||||||
|
|
||||||
func init() {
|
func InitAPI() {
|
||||||
insight := InsightAPI{}
|
insight := InsightAPI{}
|
||||||
api.HandleAPIMethod(api.POST, "/elasticsearch/:id/visualization/metadata", insight.HandleGetMetadata)
|
api.HandleAPIMethod(api.POST, "/elasticsearch/:id/visualization/metadata", insight.HandleGetMetadata)
|
||||||
api.HandleAPIMethod(api.POST, "/elasticsearch/:id/visualization/data", insight.HandleGetMetricData)
|
api.HandleAPIMethod(api.POST, "/elasticsearch/:id/visualization/data", insight.HandleGetMetricData)
|
||||||
|
|
|
@ -0,0 +1,459 @@
|
||||||
|
package task
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"golang.org/x/crypto/bcrypt"
|
||||||
|
"infini.sh/framework/core/api"
|
||||||
|
"infini.sh/framework/core/api/rbac"
|
||||||
|
httprouter "infini.sh/framework/core/api/router"
|
||||||
|
"infini.sh/framework/core/elastic"
|
||||||
|
"infini.sh/framework/core/env"
|
||||||
|
"infini.sh/framework/core/errors"
|
||||||
|
"infini.sh/framework/core/global"
|
||||||
|
"infini.sh/framework/core/module"
|
||||||
|
"infini.sh/framework/core/orm"
|
||||||
|
"infini.sh/framework/core/pipeline"
|
||||||
|
"infini.sh/framework/core/util"
|
||||||
|
elastic2 "infini.sh/framework/modules/elastic"
|
||||||
|
elastic1 "infini.sh/framework/modules/elastic/common"
|
||||||
|
elastic3 "infini.sh/framework/modules/elastic/api"
|
||||||
|
"infini.sh/framework/modules/security"
|
||||||
|
"infini.sh/framework/plugins/replay"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
uri2 "net/url"
|
||||||
|
"path"
|
||||||
|
"runtime"
|
||||||
|
"github.com/valyala/fasttemplate"
|
||||||
|
log "github.com/cihub/seelog"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Module struct {
|
||||||
|
api.Handler
|
||||||
|
}
|
||||||
|
|
||||||
|
func (module *Module) Name() string {
|
||||||
|
return "setup"
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
module.RegisterSystemModule(&Module{})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (module *Module) Setup() {
|
||||||
|
|
||||||
|
if !global.Env().SetupRequired(){
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
api.HandleAPIMethod(api.POST, "/setup/_validate", module.validate)
|
||||||
|
api.HandleAPIMethod(api.POST, "/setup/_initialize", module.initialize)
|
||||||
|
elastic3.InitTestAPI()
|
||||||
|
}
|
||||||
|
|
||||||
|
var setupFinishedCallback= []func() {}
|
||||||
|
func RegisterSetupCallback(f func()) {
|
||||||
|
setupFinishedCallback=append(setupFinishedCallback,f)
|
||||||
|
}
|
||||||
|
|
||||||
|
func InvokeSetupCallback() {
|
||||||
|
for _,v:=range setupFinishedCallback{
|
||||||
|
v()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (module *Module) Start() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (module *Module) Stop() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type SetupRequest struct {
|
||||||
|
Cluster struct {
|
||||||
|
Host string `json:"host"`
|
||||||
|
Schema string `json:"schema"`
|
||||||
|
Endpoint string `json:"endpoint"`
|
||||||
|
Username string `json:"username"`
|
||||||
|
Password string `json:"password"`
|
||||||
|
} `json:"cluster"`
|
||||||
|
|
||||||
|
BootstrapUsername string `json:"bootstrap_username"`
|
||||||
|
BootstrapPassword string `json:"bootstrap_password"`
|
||||||
|
}
|
||||||
|
|
||||||
|
var tempID="infini_default_system_cluster"
|
||||||
|
|
||||||
|
const VersionTooOld ="elasticsearch_version_too_old"
|
||||||
|
const IndicesExists ="elasticsearch_indices_exists"
|
||||||
|
const TemplateExists ="elasticsearch_template_exists"
|
||||||
|
|
||||||
|
var cfg1 elastic1.ORMConfig
|
||||||
|
|
||||||
|
func (module *Module) validate(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
|
||||||
|
|
||||||
|
if !global.Env().SetupRequired(){
|
||||||
|
module.WriteError(w, "setup not permitted", 500)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
success:=false
|
||||||
|
var err error
|
||||||
|
var errType string
|
||||||
|
var fixTips string
|
||||||
|
var code int
|
||||||
|
code=200
|
||||||
|
defer func() {
|
||||||
|
|
||||||
|
global.Env().CheckSetup()
|
||||||
|
|
||||||
|
result := util.MapStr{}
|
||||||
|
result["success"]=success
|
||||||
|
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
var v string
|
||||||
|
switch r.(type) {
|
||||||
|
case error:
|
||||||
|
v = r.(error).Error()
|
||||||
|
case runtime.Error:
|
||||||
|
v = r.(runtime.Error).Error()
|
||||||
|
case string:
|
||||||
|
v = r.(string)
|
||||||
|
}
|
||||||
|
if v!=""{
|
||||||
|
success=false
|
||||||
|
result["error"]=util.MapStr{
|
||||||
|
"reason":v,
|
||||||
|
}
|
||||||
|
if errType!=""{
|
||||||
|
result["type"]=errType
|
||||||
|
}
|
||||||
|
if fixTips!=""{
|
||||||
|
result["fix_tips"]=fixTips
|
||||||
|
}
|
||||||
|
code=500
|
||||||
|
}
|
||||||
|
}
|
||||||
|
module.WriteJSON(w, result, code)
|
||||||
|
}()
|
||||||
|
|
||||||
|
|
||||||
|
err, client,_ := module.initTempClient(r)
|
||||||
|
if err!=nil{
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
//validate version
|
||||||
|
version := client.GetVersion()
|
||||||
|
if version != "" {
|
||||||
|
ver := &util.Version{}
|
||||||
|
ver, err = util.ParseSemantic(version)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
if ver.Major() >= 7 {
|
||||||
|
if ver.Major() == 7 && ver.Minor() < 3 {
|
||||||
|
errType = VersionTooOld
|
||||||
|
panic(errors.Errorf("elasticsearch version(%v) should greater than v7.3", version))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
errType = VersionTooOld
|
||||||
|
panic(errors.Errorf("elasticsearch version(%v) should greater than v7.3", version))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cfg1 = elastic1.ORMConfig{}
|
||||||
|
exist, err := env.ParseConfig("elastic.orm", &cfg1)
|
||||||
|
if exist && err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg1.IndexPrefix==""{
|
||||||
|
cfg1.IndexPrefix=".infini_"
|
||||||
|
}
|
||||||
|
if cfg1.TemplateName==""{
|
||||||
|
cfg1.TemplateName=".infini"
|
||||||
|
}
|
||||||
|
|
||||||
|
//validate indices
|
||||||
|
indices, err := client.GetIndices(util.TrimSpaces(cfg1.IndexPrefix) + "*")
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if indices != nil && len(*indices) > 0 {
|
||||||
|
buff := bytes.Buffer{}
|
||||||
|
tipBuff := bytes.Buffer{}
|
||||||
|
for k, _ := range *indices {
|
||||||
|
buff.WriteString(k)
|
||||||
|
buff.WriteString("\n")
|
||||||
|
|
||||||
|
tipBuff.WriteString("DELETE ")
|
||||||
|
tipBuff.WriteString(k)
|
||||||
|
tipBuff.WriteString("\n")
|
||||||
|
}
|
||||||
|
errType = IndicesExists
|
||||||
|
fixTips=tipBuff.String()
|
||||||
|
panic(errors.Errorf("there are following indices exists in target elasticsearch: \n%v", buff.String()))
|
||||||
|
}
|
||||||
|
|
||||||
|
ok, err := client.TemplateExists(cfg1.TemplateName)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
if ok {
|
||||||
|
errType = TemplateExists
|
||||||
|
fixTips="DELETE /_template/"+util.TrimSpaces(cfg1.TemplateName)
|
||||||
|
panic(errors.Errorf("there are following template already exists in target elasticsearch: \n%v", cfg1.TemplateName))
|
||||||
|
}
|
||||||
|
|
||||||
|
success = true
|
||||||
|
}
|
||||||
|
var cfg elastic.ElasticsearchConfig
|
||||||
|
func (module *Module) initTempClient(r *http.Request) (error, elastic.API,SetupRequest) {
|
||||||
|
request := SetupRequest{}
|
||||||
|
err := module.DecodeJSON(r, &request)
|
||||||
|
if err != nil {
|
||||||
|
return err,nil,request
|
||||||
|
}
|
||||||
|
|
||||||
|
if request.Cluster.Endpoint==""&&request.Cluster.Host==""{
|
||||||
|
panic("invalid configuration")
|
||||||
|
}
|
||||||
|
|
||||||
|
if request.Cluster.Endpoint==""{
|
||||||
|
if request.Cluster.Host!=""&&request.Cluster.Schema!=""{
|
||||||
|
request.Cluster.Endpoint=fmt.Sprintf("%v://%v",request.Cluster.Schema,request.Cluster.Host)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg = elastic.ElasticsearchConfig{
|
||||||
|
Enabled: true,
|
||||||
|
Reserved: true,
|
||||||
|
Endpoint: request.Cluster.Endpoint,
|
||||||
|
BasicAuth: &elastic.BasicAuth{
|
||||||
|
Username: request.Cluster.Username,
|
||||||
|
Password: request.Cluster.Password,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.Endpoint!=""&&cfg.Host==""{
|
||||||
|
uri,err:=uri2.Parse(cfg.Endpoint)
|
||||||
|
if err!=nil{
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
cfg.Host=uri.Host
|
||||||
|
cfg.Schema=uri.Scheme
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.ID = tempID
|
||||||
|
cfg.Name = "INFINI_SYSTEM ("+util.PickRandomName()+")"
|
||||||
|
elastic.InitMetadata(&cfg, true)
|
||||||
|
client, err := elastic1.InitClientWithConfig(cfg)
|
||||||
|
if err != nil {
|
||||||
|
return err,nil,request
|
||||||
|
}
|
||||||
|
|
||||||
|
elastic.UpdateConfig(cfg)
|
||||||
|
elastic.UpdateClient(cfg, client)
|
||||||
|
cfg.Version=client.GetVersion()
|
||||||
|
global.Register(elastic.GlobalSystemElasticsearchID,tempID)
|
||||||
|
|
||||||
|
return err, client,request
|
||||||
|
}
|
||||||
|
|
||||||
|
func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
|
||||||
|
if !global.Env().SetupRequired(){
|
||||||
|
module.WriteError(w, "setup not permitted", 500)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
success:=false
|
||||||
|
var err error
|
||||||
|
var errType string
|
||||||
|
var fixTips string
|
||||||
|
var code int
|
||||||
|
code=200
|
||||||
|
defer func() {
|
||||||
|
|
||||||
|
global.Env().CheckSetup()
|
||||||
|
|
||||||
|
result := util.MapStr{}
|
||||||
|
result["success"]=success
|
||||||
|
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
var v string
|
||||||
|
switch r.(type) {
|
||||||
|
case error:
|
||||||
|
v = r.(error).Error()
|
||||||
|
case runtime.Error:
|
||||||
|
v = r.(runtime.Error).Error()
|
||||||
|
case string:
|
||||||
|
v = r.(string)
|
||||||
|
}
|
||||||
|
if v!=""{
|
||||||
|
success=false
|
||||||
|
result["error"]=util.MapStr{
|
||||||
|
"reason":v,
|
||||||
|
}
|
||||||
|
if errType!=""{
|
||||||
|
result["type"]=errType
|
||||||
|
}
|
||||||
|
if fixTips!=""{
|
||||||
|
result["fix_tips"]=fixTips
|
||||||
|
}
|
||||||
|
code=500
|
||||||
|
}
|
||||||
|
}
|
||||||
|
module.WriteJSON(w, result, code)
|
||||||
|
}()
|
||||||
|
|
||||||
|
err, client,request := module.initTempClient(r)
|
||||||
|
if err!=nil{
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg1.IndexPrefix==""{
|
||||||
|
cfg1.IndexPrefix=".infini_"
|
||||||
|
}
|
||||||
|
if cfg1.TemplateName==""{
|
||||||
|
cfg1.TemplateName=".infini"
|
||||||
|
}
|
||||||
|
|
||||||
|
if !cfg1.Enabled{
|
||||||
|
cfg1.Enabled=true
|
||||||
|
}
|
||||||
|
|
||||||
|
if !cfg1.InitTemplate{
|
||||||
|
cfg1.InitTemplate=true
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.Reserved=true
|
||||||
|
cfg.Monitored=true
|
||||||
|
|
||||||
|
//处理ORM
|
||||||
|
handler := elastic2.ElasticORM{Client: client, Config:cfg1 }
|
||||||
|
orm.Register("elastic_setup_"+util.GetUUID(), handler)
|
||||||
|
|
||||||
|
//处理模版
|
||||||
|
elastic2.InitTemplate(true)
|
||||||
|
|
||||||
|
//处理生命周期
|
||||||
|
//TEMPLATE_NAME
|
||||||
|
//INDEX_PREFIX
|
||||||
|
dslTplFile:=path.Join(global.Env().GetConfigDir(),"initialization.tpl")
|
||||||
|
dslFile:=path.Join(global.Env().GetConfigDir(),"initialization.dsl")
|
||||||
|
|
||||||
|
var dsl []byte
|
||||||
|
dsl,err=util.FileGetContent(dslTplFile)
|
||||||
|
if err!=nil{
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var dslWriteSuccess=false
|
||||||
|
if len(dsl)>0{
|
||||||
|
var tpl *fasttemplate.Template
|
||||||
|
tpl,err=fasttemplate.NewTemplate(string(dsl), "$[[", "]]")
|
||||||
|
if err!=nil{
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
if tpl!=nil{
|
||||||
|
output:=tpl.ExecuteFuncString(func(w io.Writer, tag string) (int, error) {
|
||||||
|
switch tag {
|
||||||
|
case "TEMPLATE_NAME":
|
||||||
|
return w.Write([]byte(cfg1.TemplateName))
|
||||||
|
case "INDEX_PREFIX":
|
||||||
|
return w.Write([]byte(cfg1.IndexPrefix))
|
||||||
|
case "RESOURCE_ID":
|
||||||
|
return w.Write([]byte(cfg.ID))
|
||||||
|
case "RESOURCE_NAME":
|
||||||
|
return w.Write([]byte(cfg.Name))
|
||||||
|
}
|
||||||
|
panic(errors.Errorf("unknown tag: %v",tag))
|
||||||
|
})
|
||||||
|
_,err=util.FilePutContent(dslFile,output)
|
||||||
|
if err!=nil{
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
dslWriteSuccess=true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if dslWriteSuccess{
|
||||||
|
lines := util.FileGetLines(dslFile)
|
||||||
|
_,err,_:=replay.ReplayLines(pipeline.AcquireContext(),lines,cfg.Schema,cfg.Host)
|
||||||
|
if err!=nil{
|
||||||
|
log.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
//处理索引
|
||||||
|
elastic2.InitSchema()
|
||||||
|
//init security
|
||||||
|
security.InitSecurity()
|
||||||
|
|
||||||
|
//保存默认集群
|
||||||
|
err=orm.Save(&cfg)
|
||||||
|
if err!=nil{
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if request.BootstrapUsername!=""&&request.BootstrapPassword!=""{
|
||||||
|
//Save bootstrap user
|
||||||
|
user:=rbac.User{}
|
||||||
|
user.ID="default_user_"+request.BootstrapUsername
|
||||||
|
user.Name=request.BootstrapUsername
|
||||||
|
user.NickName=request.BootstrapUsername
|
||||||
|
var hash []byte
|
||||||
|
hash, err = bcrypt.GenerateFromPassword([]byte(request.BootstrapPassword), bcrypt.DefaultCost)
|
||||||
|
if err!=nil{
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
user.Password=string(hash)
|
||||||
|
role:=[]rbac.UserRole{}
|
||||||
|
role=append(role,rbac.UserRole{
|
||||||
|
ID: rbac.RoleAdminName,
|
||||||
|
Name: rbac.RoleAdminName,
|
||||||
|
})
|
||||||
|
user.Roles=role
|
||||||
|
err=orm.Save(&user)
|
||||||
|
if err!=nil{
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
//save to local file
|
||||||
|
file:=path.Join(global.Env().GetConfigDir(),"system_config.yml")
|
||||||
|
_,err=util.FilePutContent(file,fmt.Sprintf("configs.template:\n - name: \"system\"\n path: ./config/system_config.tpl\n variable:\n " +
|
||||||
|
"CLUSTER_ID: %v\n CLUSTER_ENDPINT: \"%v\"\n " +
|
||||||
|
"CLUSTER_USER: \"%v\"\n CLUSTER_PASS: \"%v\"\n INDEX_PREFIX: \"%v\"",
|
||||||
|
tempID,cfg.Endpoint,cfg.BasicAuth.Username,cfg.BasicAuth.Password,cfg1.IndexPrefix ))
|
||||||
|
if err!=nil{
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
//处理 ILM
|
||||||
|
|
||||||
|
//callback
|
||||||
|
InvokeSetupCallback()
|
||||||
|
|
||||||
|
//disable builtin auth
|
||||||
|
err=api.DisableBuiltinUserAdmin()
|
||||||
|
if err!=nil{
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
//place setup lock file
|
||||||
|
setupLock:=path.Join(global.Env().GetDataDir(),".setup_lock")
|
||||||
|
_,err=util.FilePutContent(setupLock,time.Now().String())
|
||||||
|
if err!=nil{
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
success=true
|
||||||
|
|
||||||
|
}
|
6
ui.go
6
ui.go
|
@ -6,11 +6,11 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
log "github.com/cihub/seelog"
|
log "github.com/cihub/seelog"
|
||||||
|
"infini.sh/console/config"
|
||||||
|
uiapi "infini.sh/console/plugin/api"
|
||||||
"infini.sh/framework/core/api"
|
"infini.sh/framework/core/api"
|
||||||
"infini.sh/framework/core/util"
|
"infini.sh/framework/core/util"
|
||||||
"infini.sh/framework/core/vfs"
|
"infini.sh/framework/core/vfs"
|
||||||
"infini.sh/console/config"
|
|
||||||
uiapi "infini.sh/console/plugin/api"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type UI struct {
|
type UI struct {
|
||||||
|
@ -31,7 +31,7 @@ func (h UI) InitUI() {
|
||||||
//
|
//
|
||||||
//api.HandleUIFunc("/config", func(w http.ResponseWriter, req *http.Request){
|
//api.HandleUIFunc("/config", func(w http.ResponseWriter, req *http.Request){
|
||||||
// if(strings.TrimSpace(apiEndpoint) == ""){
|
// if(strings.TrimSpace(apiEndpoint) == ""){
|
||||||
// hostParts := strings.Split(req.Host, ":")
|
// hostParts := strings.Split(req.RemoteIP, ":")
|
||||||
// apiEndpoint = fmt.Sprintf("%s//%s:%s", apiConfig.GetSchema(), hostParts[0], apiConfig.NetworkConfig.GetBindingPort())
|
// apiEndpoint = fmt.Sprintf("%s//%s:%s", apiConfig.GetSchema(), hostParts[0], apiConfig.NetworkConfig.GetBindingPort())
|
||||||
// }
|
// }
|
||||||
// buf, _ := json.Marshal(util.MapStr{
|
// buf, _ := json.Marshal(util.MapStr{
|
||||||
|
|
Loading…
Reference in New Issue