diff --git a/main.go b/main.go index 410c5572..803e5e83 100644 --- a/main.go +++ b/main.go @@ -6,6 +6,8 @@ import ( "infini.sh/console/config" "infini.sh/console/model" "infini.sh/console/model/alerting" + "infini.sh/console/model/gateway" + _ "infini.sh/console/plugin" alertSrv "infini.sh/console/service/alerting" "infini.sh/framework" "infini.sh/framework/core/elastic" @@ -25,7 +27,6 @@ import ( "infini.sh/framework/plugins/elastic/json_indexing" api2 "infini.sh/gateway/api" _ "infini.sh/gateway/proxy" - _ "infini.sh/console/plugin" ) var appConfig *config.AppConfig @@ -120,6 +121,9 @@ func main() { orm.RegisterSchemaWithIndexName(alerting.AlertingHistory{}, "alerting-history") orm.RegisterSchemaWithIndexName(elastic.CommonCommand{}, "commands") orm.RegisterSchemaWithIndexName(elastic.TraceTemplate{}, "trace-template") + orm.RegisterSchemaWithIndexName(gateway.Instance{} , "gateway-instance") + orm.RegisterSchemaWithIndexName(gateway.Group{} , "gateway-group") + orm.RegisterSchemaWithIndexName(gateway.InstanceGroup{} , "gateway-instance-group") api.RegisterSchema() diff --git a/model/gateway/group.go b/model/gateway/group.go new file mode 100644 index 00000000..b98294b9 --- /dev/null +++ b/model/gateway/group.go @@ -0,0 +1,13 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package gateway + +import "infini.sh/framework/core/orm" + +type Group struct { + orm.ORMObjectBase + Name string `json:"name,omitempty" elastic_mapping:"name:{type:keyword,fields:{text: {type: text}}}"` + Owner string `json:"owner,omitempty" config:"owner" elastic_mapping:"owner:{type:keyword}"` +} diff --git a/model/gateway/group_instance.go b/model/gateway/group_instance.go new file mode 100644 index 00000000..c4aae676 --- /dev/null +++ b/model/gateway/group_instance.go @@ -0,0 +1,13 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package gateway + +import "infini.sh/framework/core/orm" + +type InstanceGroup struct { + orm.ORMObjectBase + GroupID string `json:"group_id,omitempty" elastic_mapping:"group_id: { type: keyword }"` + InstanceID string `json:"instance_id,omitempty" elastic_mapping:"instance_id: { type: keyword }"` +} \ No newline at end of file diff --git a/model/gateway/instance.go b/model/gateway/instance.go new file mode 100644 index 00000000..66090d60 --- /dev/null +++ b/model/gateway/instance.go @@ -0,0 +1,26 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package gateway + +import ( + "infini.sh/framework/core/orm" +) + + +type Instance struct { + orm.ORMObjectBase + + InstanceID string `json:"instance_id,omitempty" elastic_mapping:"instance_id: { type: keyword }"` + Name string `json:"name,omitempty" elastic_mapping:"name:{type:keyword,fields:{text: {type: text}}}"` + Endpoint string `json:"endpoint,omitempty" elastic_mapping:"endpoint: { type: keyword }"` + Version map[string]interface{} `json:"version,omitempty" elastic_mapping:"version: { type: object }"` + BasicAuth *struct { + Username string `json:"username,omitempty" config:"username" elastic_mapping:"username:{type:keyword}"` + Password string `json:"password,omitempty" config:"password" elastic_mapping:"password:{type:keyword}"` + } `config:"basic_auth" json:"basic_auth,omitempty" elastic_mapping:"basic_auth:{type:object}"` + Owner string `json:"owner,omitempty" config:"owner" elastic_mapping:"owner:{type:keyword}"` + Group string `json:"group,omitempty"` + Description string `json:"description,omitempty" config:"description" elastic_mapping:"description:{type:keyword}"` +} diff --git a/plugin/api/gateway/api.go b/plugin/api/gateway/api.go new file mode 100644 index 00000000..591d3420 --- /dev/null +++ b/plugin/api/gateway/api.go @@ -0,0 +1,26 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package gateway + +import ( + "infini.sh/framework/core/api" +) + +type GatewayAPI struct { + api.Handler +} + +func init() { + gateway:=GatewayAPI{} + api.HandleAPIMethod(api.POST, "/gateway/instance/try_connect", gateway.tryConnect) + api.HandleAPIMethod(api.GET, "/gateway/instance/:instance_id", gateway.getInstance) + api.HandleAPIMethod(api.POST, "/gateway/instance", gateway.createInstance) + api.HandleAPIMethod(api.PUT, "/gateway/instance/:instance_id", gateway.updateInstance) + api.HandleAPIMethod(api.DELETE, "/gateway/instance/:instance_id", gateway.deleteInstance) + api.HandleAPIMethod(api.GET, "/gateway/instance/_search", gateway.searchInstance) + + api.HandleAPIMethod(api.GET, "/gateway/group/:group_id", gateway.getGroup) + api.HandleAPIMethod(api.GET, "/gateway/group/_search", gateway.searchGroup) +} diff --git a/plugin/api/gateway/group.go b/plugin/api/gateway/group.go new file mode 100644 index 00000000..acbcb892 --- /dev/null +++ b/plugin/api/gateway/group.go @@ -0,0 +1,77 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package gateway + +import ( + "fmt" + "infini.sh/console/model/gateway" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "net/http" + log "src/github.com/cihub/seelog" + "strconv" + "strings" +) + +func (h *GatewayAPI) getGroup(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("group_id") + + obj := gateway.Group{} + obj.ID = id + + exists, err := orm.Get(&obj) + if !exists || err != nil { + h.WriteJSON(w, util.MapStr{ + "_id": id, + "found": false, + }, http.StatusNotFound) + return + } + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + h.WriteJSON(w, util.MapStr{ + "found": true, + "_id": id, + "_source": obj, + }, 200) +} + +func (h *GatewayAPI) searchGroup(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + + var ( + name = h.GetParameterOrDefault(req, "name", "") + queryDSL = `{"query":{"bool":{"must":[%s]}}, "size": %d, "from": %d}` + strSize = h.GetParameterOrDefault(req, "size", "20") + strFrom = h.GetParameterOrDefault(req, "from", "0") + mustBuilder = &strings.Builder{} + ) + if name != "" { + mustBuilder.WriteString(fmt.Sprintf(`{"prefix":{"name.text": "%s"}}`, name)) + } + size, _ := strconv.Atoi(strSize) + if size <= 0 { + size = 20 + } + from, _ := strconv.Atoi(strFrom) + if from < 0 { + from = 0 + } + + q := orm.Query{} + queryDSL = fmt.Sprintf(queryDSL, mustBuilder.String(), size, from) + q.RawQuery = []byte(queryDSL) + + err, res := orm.Search(&gateway.Group{}, &q) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.Write(w, res.Raw) +} \ No newline at end of file diff --git a/plugin/api/gateway/helper.go b/plugin/api/gateway/helper.go new file mode 100644 index 00000000..adb525cb --- /dev/null +++ b/plugin/api/gateway/helper.go @@ -0,0 +1,90 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package gateway + +import ( + "fmt" + "infini.sh/console/model/gateway" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" +) + +func fetchInstanceGroup(instanceID string) (string, error){ + // fetch gateway instance group + q := orm.Query{} + q.RawQuery = []byte(fmt.Sprintf(`{"size": 1, "query":{"term":{"instance_id":{"value":"%s"}}}}`, instanceID)) + err, res := orm.Search(&gateway.InstanceGroup{}, &q) + if err != nil { + return "", err + } + if len(res.Result) > 0 { + if rowMap, ok := res.Result[0].(map[string]interface{}); ok { + return rowMap["group_id"].(string), nil + } + } + return "", nil +} + +func fetchInstanceGroupByID(instanceIDs []interface{})([]interface{}, error){ + if len(instanceIDs) == 0 { + return nil, nil + } + // fetch gateway instance groups + esQuery := util.MapStr{ + "query": util.MapStr{ + "terms": util.MapStr{ + "instance_id": instanceIDs, + }, + }, + } + q := orm.Query{} + q.RawQuery = util.MustToJSONBytes(esQuery) + err, res := orm.Search(&gateway.InstanceGroup{}, &q) + return res.Result, err +} +func fetchGroupByID(groupIDs []interface{})([]interface{}, error){ + if len(groupIDs) == 0 { + return nil, nil + } + // fetch gateway groups + esQuery := util.MapStr{ + "query": util.MapStr{ + "terms": util.MapStr{ + "_id": groupIDs, + }, + }, + } + q := orm.Query{} + q.RawQuery = util.MustToJSONBytes(esQuery) + err, res := orm.Search(&gateway.Group{}, &q) + return res.Result, err +} + +func pickElasticsearchColumnValues(result []interface{}, columnName string) []interface{}{ + if len(result) == 0 { + return nil + } + columnValues := make([]interface{}, 0, len(result)) + for _, row := range result { + if rowMap, ok := row.(map[string]interface{}); ok { + columnValues = append(columnValues, rowMap[columnName]) + } + } + return columnValues +} + +func getRelationshipMap(result []interface{}, key string, value string) map[string]interface{}{ + if len(result) == 0 { + return nil + } + resultMap := map[string]interface{}{} + for _, row := range result { + if rowMap, ok := row.(map[string]interface{}); ok { + resultMap[rowMap[key].(string)] = rowMap[value] + } + } + return resultMap +} + diff --git a/plugin/api/gateway/instance.go b/plugin/api/gateway/instance.go new file mode 100644 index 00000000..66d987d5 --- /dev/null +++ b/plugin/api/gateway/instance.go @@ -0,0 +1,336 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package gateway + +import ( + "crypto/tls" + "fmt" + "github.com/segmentio/encoding/json" + "infini.sh/console/model/gateway" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "infini.sh/framework/lib/fasthttp" + "net/http" + log "src/github.com/cihub/seelog" + "strconv" + "strings" + "time" +) + +func (h *GatewayAPI) createInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var obj = &gateway.Instance{} + err := h.DecodeJSON(req, obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + + var groupID = obj.Group + obj.Group = "" + err = orm.Create(obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + if groupID != "" { + groupInstance := &gateway.InstanceGroup{} + groupInstance.InstanceID = obj.ID + groupInstance.GroupID = groupID + err = orm.Create(groupInstance) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + } + + h.WriteJSON(w, util.MapStr{ + "_id": obj.ID, + "result": "created", + }, 200) + +} + +func (h *GatewayAPI) getInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("instance_id") + + obj := gateway.Instance{} + obj.ID = id + + exists, err := orm.Get(&obj) + if !exists || err != nil { + h.WriteJSON(w, util.MapStr{ + "_id": id, + "found": false, + }, http.StatusNotFound) + return + } + obj.Group, err = fetchInstanceGroup(id) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + + h.WriteJSON(w, util.MapStr{ + "found": true, + "_id": id, + "_source": obj, + }, 200) +} + +func (h *GatewayAPI) updateInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("instance_id") + obj := gateway.Instance{} + + obj.ID = id + exists, err := orm.Get(&obj) + if !exists || err != nil { + h.WriteJSON(w, util.MapStr{ + "_id": id, + "result": "not_found", + }, http.StatusNotFound) + return + } + oldGroup, err := fetchInstanceGroup(id) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + + id = obj.ID + create := obj.Created + obj = gateway.Instance{} + err = h.DecodeJSON(req, &obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + if obj.Group != oldGroup { + delQuery := util.MapStr{ + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + //{ + // "term": util.MapStr{ + // "group_id": util.MapStr{ + // "value": oldGroup, + // }, + // }, + //}, + { + "term": util.MapStr{ + "instance_id": util.MapStr{ + "value": id, + }, + }, + }, + }, + }, + }, + } + err = orm.DeleteBy(&gateway.InstanceGroup{}, util.MustToJSONBytes(delQuery)) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + err = orm.Create(&gateway.InstanceGroup{ + GroupID: obj.Group, + InstanceID: id, + }) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + } + + //protect + obj.ID = id + obj.Created = create + obj.Group = "" + err = orm.Update(&obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + + h.WriteJSON(w, util.MapStr{ + "_id": obj.ID, + "result": "updated", + }, 200) +} + +func (h *GatewayAPI) deleteInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("instance_id") + + obj := gateway.Instance{} + obj.ID = id + + exists, err := orm.Get(&obj) + if !exists || err != nil { + h.WriteJSON(w, util.MapStr{ + "_id": id, + "result": "not_found", + }, http.StatusNotFound) + return + } + + err = orm.Delete(&obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + + h.WriteJSON(w, util.MapStr{ + "_id": obj.ID, + "result": "deleted", + }, 200) +} + +func (h *GatewayAPI) searchInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + + var ( + name = h.GetParameterOrDefault(req, "name", "") + queryDSL = `{"query":{"bool":{"must":[%s]}}, "size": %d, "from": %d}` + strSize = h.GetParameterOrDefault(req, "size", "20") + strFrom = h.GetParameterOrDefault(req, "from", "0") + mustBuilder = &strings.Builder{} + ) + if name != "" { + mustBuilder.WriteString(fmt.Sprintf(`{"prefix":{"name": "%s"}}`, name)) + } + size, _ := strconv.Atoi(strSize) + if size <= 0 { + size = 20 + } + from, _ := strconv.Atoi(strFrom) + if from < 0 { + from = 0 + } + + q := orm.Query{} + queryDSL = fmt.Sprintf(queryDSL, mustBuilder.String(), size, from) + q.RawQuery = []byte(queryDSL) + + err, res := orm.Search(&gateway.Instance{}, &q) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + //fetch relationship + instanceIDs := pickElasticsearchColumnValues(res.Result, "id") + instanceGroups, err := fetchInstanceGroupByID(instanceIDs) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + //fetch group + + groupIDs := pickElasticsearchColumnValues(instanceGroups, "group_id") + groups, err := fetchGroupByID(groupIDs) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + groupsMap := getRelationshipMap(groups, "id", "name") + relationshipMap := getRelationshipMap(instanceGroups, "instance_id", "group_id") + + resultRes := &elastic.SearchResponse{} + err = util.FromJSONBytes(res.Raw, resultRes) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + for _, hit := range resultRes.Hits.Hits { + hit.Source["group"] = relationshipMap[hit.ID] + } + + h.WriteJSON(w, struct{ + elastic.SearchResponse + Groups interface{} `json:"groups"` + }{ + SearchResponse: *resultRes, + Groups: groupsMap, + }, http.StatusOK) +} + +type GatewayConnectResponse struct { + ID string `json:"id"` + Name string `json:"name"` + Tagline string `json:"tagline"` + Version struct { + BuildDate string `json:"build_date"` + BuildHash string `json:"build_hash"` + EOLDate string `json:"eol_date"` + Number string `json:"number"` + } `json:"version"` + +} +func (h *GatewayAPI) doConnect(endpoint, username, password string) (*GatewayConnectResponse, error) { + var ( + freq = fasthttp.AcquireRequest() + fres = fasthttp.AcquireResponse() + ) + defer func() { + fasthttp.ReleaseRequest(freq) + fasthttp.ReleaseResponse(fres) + }() + + freq.SetRequestURI(fmt.Sprintf("%s/_framework/api/_info", endpoint)) + freq.Header.SetMethod("GET") + if username != ""{ + freq.SetBasicAuth(username, password) + } + + client := &fasthttp.Client{ + MaxConnsPerHost: 1000, + TLSConfig: &tls.Config{InsecureSkipVerify: true}, + ReadTimeout: time.Second * 5, + } + err := client.Do(freq, fres) + if err != nil { + return nil, err + } + b := fres.Body() + gres := &GatewayConnectResponse{} + err = json.Unmarshal(b, gres) + return gres, err + +} + +func (h *GatewayAPI) tryConnect(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var reqBody = struct { + Endpoint string `json:"endpoint"` + BasicAuth struct { + Username string `json:"username"` + Password string `json:"password"` + } + }{} + err := h.DecodeJSON(req, &reqBody) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + connectRes, err := h.doConnect(reqBody.Endpoint, reqBody.BasicAuth.Username, reqBody.BasicAuth.Password) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.WriteJSON(w, connectRes, http.StatusOK) +} \ No newline at end of file