From 32ed24cee0dea5089c51de5d74410c1732afca01 Mon Sep 17 00:00:00 2001 From: liugq Date: Fri, 16 Jun 2023 17:30:11 +0800 Subject: [PATCH] add render label template api --- common/elastic.go | 85 ++++++++++++++++++++++++++++++++- model/insight/metric_data.go | 7 +++ model/insight/visualization.go | 4 +- plugin/api/insight/api.go | 2 +- plugin/api/insight/map_label.go | 63 ++++++++++++++++++++++++ plugin/api/insight/metadata.go | 54 +-------------------- 6 files changed, 158 insertions(+), 57 deletions(-) create mode 100644 plugin/api/insight/map_label.go diff --git a/common/elastic.go b/common/elastic.go index 45761853..fbfe152a 100644 --- a/common/elastic.go +++ b/common/elastic.go @@ -5,10 +5,15 @@ package common import ( + "bytes" "fmt" + log "github.com/cihub/seelog" "infini.sh/framework/core/elastic" "infini.sh/framework/core/orm" "infini.sh/framework/core/util" + "sync" + "text/template" + "time" ) //GetClusterNames query cluster names by cluster ids @@ -85,4 +90,82 @@ func GetMapStringValue(m util.MapStr, key string) string { return "" } return util.ToString(v) -} \ No newline at end of file +} + +func MapLabel(labelName, indexName, indexKeyField, indexValueField string, client elastic.API) string { + labelMaps, err := getOrInitLabelCache(indexName, indexKeyField, indexValueField, client) + if err != nil { + log.Error(err) + return "" + } + return labelMaps[labelName] +} + +var labelCache = sync.Map{} +type LabelCacheItem struct { + KeyValues map[string]string + Timestamp time.Time +} +func getOrInitLabelCache(indexName, indexKeyField, indexValueField string, client elastic.API) (map[string]string, error){ + cacheKey := fmt.Sprintf("%s_%s_%s", indexName, indexKeyField, indexValueField ) + var ( + labelMaps = map[string]string{} + err error + ) + if v, ok := labelCache.Load(cacheKey); ok { + if cacheItem, ok := v.(*LabelCacheItem); ok { + if cacheItem.Timestamp.Add(time.Minute).After(time.Now()){ + return cacheItem.KeyValues, nil + } + //cache expired + } + } + labelMaps, err = getLabelMaps(indexName, indexKeyField, indexValueField, client) + if err != nil { + return labelMaps, err + } + labelCache.Store(cacheKey, &LabelCacheItem{ + KeyValues: labelMaps, + Timestamp: time.Now(), + }) + return labelMaps, nil +} + +func getLabelMaps( indexName, indexKeyField, indexValueField string, client elastic.API) (map[string]string, error){ + if client == nil { + return nil, fmt.Errorf("cluster client must not be empty") + } + query := util.MapStr{ + "size": 1000, + "collapse": util.MapStr{ + "field": indexKeyField, + }, + "_source": []string{indexKeyField, indexValueField}, + } + queryDsl := util.MustToJSONBytes(query) + searchRes, err := client.SearchWithRawQueryDSL(indexName, queryDsl) + if err != nil { + return nil, err + } + labelMaps := map[string]string{} + for _, hit := range searchRes.Hits.Hits { + sourceM := util.MapStr(hit.Source) + v := GetMapStringValue(sourceM, indexValueField) + var key string + if indexKeyField == "_id" { + key = hit.ID + }else{ + key = GetMapStringValue(sourceM, indexKeyField) + } + if key != "" { + labelMaps[key] = v + } + } + return labelMaps, nil +} + +func ExecuteTemplate( tpl *template.Template, ctx map[string]interface{}) ([]byte, error){ + msgBuffer := &bytes.Buffer{} + err := tpl.Execute(msgBuffer, ctx) + return msgBuffer.Bytes(), err +} diff --git a/model/insight/metric_data.go b/model/insight/metric_data.go index 180488ad..7125eeb8 100644 --- a/model/insight/metric_data.go +++ b/model/insight/metric_data.go @@ -22,6 +22,7 @@ type Metric struct { FormatType string `json:"format_type,omitempty"` TimeFilter interface{} `json:"time_filter,omitempty"` TimeBeforeGroup bool `json:"time_before_group,omitempty"` + BucketLabel *BucketLabel `json:"bucket_label,omitempty"` } type MetricGroupItem struct { @@ -71,10 +72,16 @@ type MetricDataItem struct { Timestamp interface{} `json:"timestamp,omitempty"` Value interface{} `json:"value"` Groups []string `json:"groups,omitempty"` + GroupLabel string `json:"group_label,omitempty"` } type MetricData struct { Groups []string `json:"groups,omitempty"` Data map[string][]MetricDataItem + GroupLabel string `json:"group_label,omitempty"` } +type BucketLabel struct { + Enabled bool `json:"enabled"` + Template string `json:"template,omitempty"` +} diff --git a/model/insight/visualization.go b/model/insight/visualization.go index aba3045a..6cd14c03 100644 --- a/model/insight/visualization.go +++ b/model/insight/visualization.go @@ -13,8 +13,8 @@ type Visualization struct { Title string `json:"title,omitempty" elastic_mapping:"title: { type: keyword }"` IndexPattern string `json:"index_pattern,omitempty" elastic_mapping:"index_pattern: { type: keyword }"` ClusterId string `json:"cluster_id,omitempty" elastic_mapping:"cluster_id: { type: keyword }"` - Series []SeriesItem `json:"series" elastic_mapping:"series: { type: object }"` - Position *Position `json:"position,omitempty" elastic_mapping:"position: { type: object }"` + Series []SeriesItem `json:"series" elastic_mapping:"series: { type: object,enabled:false }"` + Position *Position `json:"position,omitempty" elastic_mapping:"position: { type: object,enabled:false }"` Description string `json:"description,omitempty" elastic_mapping:"description: { type: keyword }"` } diff --git a/plugin/api/insight/api.go b/plugin/api/insight/api.go index f050a45e..e3c9ed82 100644 --- a/plugin/api/insight/api.go +++ b/plugin/api/insight/api.go @@ -27,5 +27,5 @@ func InitAPI() { api.HandleAPIMethod(api.PUT, "/insight/dashboard/:dashboard_id", insight.updateDashboard) api.HandleAPIMethod(api.DELETE, "/insight/dashboard/:dashboard_id", insight.deleteDashboard) api.HandleAPIMethod(api.GET, "/insight/dashboard/_search", insight.searchDashboard) - + api.HandleAPIMethod(api.POST, "/elasticsearch/:id/map_label/_render", insight.renderMapLabelTemplate) } diff --git a/plugin/api/insight/map_label.go b/plugin/api/insight/map_label.go new file mode 100644 index 00000000..03d97bc1 --- /dev/null +++ b/plugin/api/insight/map_label.go @@ -0,0 +1,63 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package insight + +import ( + log "github.com/cihub/seelog" + common2 "infini.sh/console/common" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/util" + "net/http" + "text/template" +) + +func (h *InsightAPI) renderMapLabelTemplate(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + clusterID := ps.MustGetParameter("id") + body := &RenderTemplateRequest{} + err := h.DecodeJSON(req, &body) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if len(body.Contexts) == 0 && body.Template == "" { + log.Error("got bad request body: %v", body) + h.WriteError(w, "bad request", http.StatusInternalServerError) + } + client := elastic.GetClient(clusterID) + tpl, err := template.New("template_render").Funcs(map[string]any{ + "map_label": func(indexName, indexKeyField, indexValueField, labelName string) string { + return common2.MapLabel(labelName, indexName, indexKeyField, indexValueField, client) + }, + }).Parse(body.Template) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + resultLabels := map[string]string{} + for _, ctx := range body.Contexts { + label, err := common2.ExecuteTemplate(tpl, ctx.Value) + if err != nil { + log.Error(err) + continue + } + resultLabels[ctx.Key] = string(label) + } + h.WriteJSON(w, util.MapStr{ + "labels": resultLabels, + }, http.StatusOK) +} + +type RenderTemplateRequest struct { + Contexts []RenderTemplateContext `json:"contexts"` + Template string `json:"template"` +} + +type RenderTemplateContext struct { + Key string `json:"key"` + Value map[string]interface{} `json:"value"` +} \ No newline at end of file diff --git a/plugin/api/insight/metadata.go b/plugin/api/insight/metadata.go index e5dbf381..81b2de18 100644 --- a/plugin/api/insight/metadata.go +++ b/plugin/api/insight/metadata.go @@ -7,7 +7,6 @@ package insight import ( "github.com/Knetic/govaluate" log "github.com/cihub/seelog" - common2 "infini.sh/console/common" "infini.sh/console/model/insight" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" @@ -278,60 +277,9 @@ func getMetricData(metric *insight.Metric) (interface{}, error) { targetMetricData = append(targetMetricData, targetData) } } + result := []insight.MetricDataItem{} - //transform cluster_id, node_id to name - var ( - clusterIDsM = map[string]struct{}{} - nodeIDsM = map[string]struct{}{} - ) for _, md := range targetMetricData { - for i, gv := range md.Groups { - switch metric.Groups[i].Field { - case "metadata.labels.cluster_id", "metadata.cluster_id": - clusterIDsM[gv] = struct{}{} - case "metadata.node_id", "metadata.labels.node_id": - nodeIDsM[gv] = struct{}{} - default: - } - } - } - var ( - clusterIDs []string - nodeIDs []string - clusterIDToNames = map[string]string{} - nodeIDToNames = map[string]string{} - ) - if len(clusterIDsM) > 0 { - for k, _ := range clusterIDsM { - clusterIDs = append(clusterIDs, k) - } - clusterIDToNames, err = common2.GetClusterNames(clusterIDs) - if err != nil { - return nil, err - } - } - if len(nodeIDsM) > 0 { - for k, _ := range nodeIDsM { - nodeIDs = append(nodeIDs, k) - } - nodeIDToNames, err = common2.GetNodeNames(nodeIDs) - if err != nil { - return nil, err - } - } - for _, md := range targetMetricData { - for i, gv := range md.Groups { - switch metric.Groups[i].Field { - case "metadata.labels.cluster_id", "metadata.cluster_id": - if name, ok := clusterIDToNames[gv]; ok && name != "" { - md.Groups[i] = name - } - case "metadata.node_id", "metadata.labels.node_id": - if name, ok := nodeIDToNames[gv]; ok && name != "" { - md.Groups[i] = name - } - } - } for _, v := range md.Data { for _, mitem := range v { mitem.Groups = md.Groups