floraachy
/
console
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

optimize map label api

This commit is contained in:
liugq 2023-06-19 07:21:22 +08:00
parent 68482f5fec
commit 1fe5ace58d
4 changed files with 139 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

132
common/elastic.go
View File

@ -9,81 +9,10 @@ import (
"fmt"
log "github.com/cihub/seelog"
"infini.sh/framework/core/elastic"
"infini.sh/framework/core/orm"
"infini.sh/framework/core/util"
"sync"
"text/template"
"time"
)
//GetClusterNames query cluster names by cluster ids
func GetClusterNames(clusterIds []string) (map[string]string, error){
if len(clusterIds) == 0 {
return nil, fmt.Errorf("cluster ids must not be empty")
}
rq := util.MapStr{
"size": len(clusterIds),
"query": util.MapStr{
"terms": util.MapStr{
"id": clusterIds,
},
},
}
idToNames := map[string]string{}
q := orm.Query{
RawQuery: util.MustToJSONBytes(rq),
}
err, result := orm.Search(elastic.ElasticsearchConfig{}, &q)
if err != nil {
return nil, err
}
for _, row := range result.Result {
if rowM, ok := row.(map[string]interface{}); ok {
if id, ok := rowM["id"].(string); ok {
if name, ok := rowM["name"].(string); ok {
idToNames[id] = name
}
}
}
}
return idToNames, nil
}
//GetNodeNames query node names by node ids
func GetNodeNames(nodeIDs []string) (map[string]string, error){
if len(nodeIDs) == 0 {
return nil, fmt.Errorf("node ids must not be empty")
}
rq := util.MapStr{
"size": 500,
"query": util.MapStr{
"terms": util.MapStr{
"metadata.node_id": nodeIDs,
},
},
}
idToNames := map[string]string{}
q := orm.Query{
RawQuery: util.MustToJSONBytes(rq),
}
err, result := orm.Search(elastic.NodeConfig{}, &q)
if err != nil {
return nil, err
}
for _, row := range result.Result {
if rowM, ok := row.(map[string]interface{}); ok {
id := GetMapStringValue(rowM, "metadata.node_id")
name := GetMapStringValue(rowM, "metadata.node_name")
if id != "" {
idToNames[id] = name
}
}
}
return idToNames, nil
}
func GetMapStringValue(m util.MapStr, key string) string {
v, err := m.GetValue(key)
if err != nil {
@ -92,8 +21,13 @@ func GetMapStringValue(m util.MapStr, key string) string {
return util.ToString(v)
}
func MapLabel(labelName, indexName, indexKeyField, indexValueField string, client elastic.API) string {
labelMaps, err := getOrInitLabelCache(indexName, indexKeyField, indexValueField, client)
func MapLabel(labelName, indexName, keyField, valueField string, client elastic.API, cacheLabels map[string]string) string {
if len(cacheLabels) > 0 {
if v, ok := cacheLabels[labelName]; ok{
return v
}
}
labelMaps, err := GetLabelMaps(indexName, keyField, valueField, client, []string{labelName}, 1)
if err != nil {
log.Error(err)
return ""
@ -101,46 +35,24 @@ func MapLabel(labelName, indexName, indexKeyField, indexValueField string, clien
return labelMaps[labelName]
}
var labelCache = sync.Map{}
type LabelCacheItem struct {
KeyValues map[string]string
Timestamp time.Time
}
func getOrInitLabelCache(indexName, indexKeyField, indexValueField string, client elastic.API) (map[string]string, error){
cacheKey := fmt.Sprintf("%s_%s_%s", indexName, indexKeyField, indexValueField )
var (
labelMaps = map[string]string{}
err error
)
if v, ok := labelCache.Load(cacheKey); ok {
if cacheItem, ok := v.(*LabelCacheItem); ok {
if cacheItem.Timestamp.Add(time.Minute).After(time.Now()){
return cacheItem.KeyValues, nil
}
//cache expired
}
}
labelMaps, err = getLabelMaps(indexName, indexKeyField, indexValueField, client)
if err != nil {
return labelMaps, err
}
labelCache.Store(cacheKey, &LabelCacheItem{
KeyValues: labelMaps,
Timestamp: time.Now(),
})
return labelMaps, nil
}
func getLabelMaps( indexName, indexKeyField, indexValueField string, client elastic.API) (map[string]string, error){
func GetLabelMaps( indexName, keyField, valueField string, client elastic.API, keyFieldValues []string, cacheSize int) (map[string]string, error){
if client == nil {
return nil, fmt.Errorf("cluster client must not be empty")
}
query := util.MapStr{
"size": 1000,
"size": cacheSize,
"collapse": util.MapStr{
"field": indexKeyField,
"field": keyField,
},
"_source": []string{indexKeyField, indexValueField},
"_source": []string{keyField, valueField},
}
if len(keyFieldValues) > 0 {
query["query"] = util.MapStr{
"terms": util.MapStr{
keyField: keyFieldValues,
},
}
query["size"] = len(keyFieldValues)
}
queryDsl := util.MustToJSONBytes(query)
searchRes, err := client.SearchWithRawQueryDSL(indexName, queryDsl)
@ -150,12 +62,12 @@ func getLabelMaps( indexName, indexKeyField, indexValueField string, client elas
labelMaps := map[string]string{}
for _, hit := range searchRes.Hits.Hits {
sourceM := util.MapStr(hit.Source)
v := GetMapStringValue(sourceM, indexValueField)
v := GetMapStringValue(sourceM, valueField)
var key string
if indexKeyField == "_id" {
if keyField == "_id" {
key = hit.ID
}else{
key = GetMapStringValue(sourceM, indexKeyField)
key = GetMapStringValue(sourceM, keyField)
}
if key != "" {
labelMaps[key] = v

8
model/alerting/metric.go
View File

@ -7,6 +7,7 @@ package alerting
import (
"fmt"
"infini.sh/console/model/insight"
"infini.sh/framework/modules/elastic/common"
"regexp"
)
@ -59,4 +60,9 @@ type MetricData struct {
Data map[string][]TimeMetricData `json:"data"`
}
type TimeMetricData []interface{}
type TimeMetricData []interface{}
type AlertMetricItem struct {
common.MetricItem
BucketGroups [][]string `json:"bucket_groups,omitempty"`
}

141
plugin/api/alerting/rule.go
View File

@ -9,7 +9,6 @@ import (
"fmt"
log "github.com/cihub/seelog"
"github.com/r3labs/diff/v2"
common2 "infini.sh/console/common"
"infini.sh/console/model/alerting"
"infini.sh/console/model/insight"
alerting2 "infini.sh/console/service/alerting"
@ -784,6 +783,7 @@ func (alertAPI *AlertAPI) getMetricData(w http.ResponseWriter, req *http.Request
}
resBody := util.MapStr{
"metric": metricItem,
"bucket_label": rule.Metrics.BucketLabel,
}
if alertAPI.GetParameter(req, "debug") == "1" {
resBody["query"] = queryResult.Query
@ -791,7 +791,7 @@ func (alertAPI *AlertAPI) getMetricData(w http.ResponseWriter, req *http.Request
alertAPI.WriteJSON(w,resBody, http.StatusOK)
}
func getRuleMetricData( rule *alerting.Rule, filterParam *alerting.FilterParam) (*common.MetricItem, *alerting.QueryResult, error) {
func getRuleMetricData( rule *alerting.Rule, filterParam *alerting.FilterParam) (*alerting.AlertMetricItem, *alerting.QueryResult, error) {
eng := alerting2.GetEngine(rule.Resource.Type)
metricData, queryResult, err := eng.GetTargetMetricData(rule, true, filterParam)
if err != nil {
@ -802,54 +802,56 @@ func getRuleMetricData( rule *alerting.Rule, filterParam *alerting.FilterParam)
if rule.Metrics.FormatType != "" {
formatType = rule.Metrics.FormatType
}
var metricItem = common.MetricItem{
Group: rule.ID,
Key: rule.ID,
Axis: []*common.MetricAxis{
{ID: util.GetUUID(), Group: rule.ID, Title: "", FormatType: formatType, Position: "left", ShowGridLines: true,
TickFormat: "0,0.[00]",
Ticks: 5},
var metricItem = alerting.AlertMetricItem{
MetricItem: common.MetricItem{
Group: rule.ID,
Key: rule.ID,
Axis: []*common.MetricAxis{
{ID: util.GetUUID(), Group: rule.ID, Title: "", FormatType: formatType, Position: "left", ShowGridLines: true,
TickFormat: "0,0.[00]",
Ticks: 5},
},
},
}
var (
clusterIDsM = map[string]struct{}{}
nodeIDsM = map[string]struct{}{}
)
for _, md := range metricData {
for i, gv := range md.GroupValues {
switch rule.Metrics.Groups[i].Field {
case "metadata.labels.cluster_id", "metadata.cluster_id":
clusterIDsM[gv] = struct{}{}
case "metadata.node_id", "metadata.labels.node_id":
nodeIDsM[gv] = struct{}{}
default:
}
}
}
var (
clusterIDs []string
nodeIDs []string
clusterIDToNames = map[string]string{}
nodeIDToNames = map[string]string{}
)
if len(clusterIDsM) > 0 {
for k, _ := range clusterIDsM {
clusterIDs = append(clusterIDs, k)
}
clusterIDToNames, err = common2.GetClusterNames(clusterIDs)
if err != nil {
return nil,queryResult, err
}
}
if len(nodeIDsM) > 0 {
for k, _ := range nodeIDsM {
nodeIDs = append(nodeIDs, k)
}
nodeIDToNames, err = common2.GetNodeNames(nodeIDs)
if err != nil {
return nil,queryResult, err
}
}
//var (
// clusterIDsM = map[string]struct{}{}
// nodeIDsM = map[string]struct{}{}
//)
//for _, md := range metricData {
// for i, gv := range md.GroupValues {
// switch rule.Metrics.Groups[i].Field {
// case "metadata.labels.cluster_id", "metadata.cluster_id":
// clusterIDsM[gv] = struct{}{}
// case "metadata.node_id", "metadata.labels.node_id":
// nodeIDsM[gv] = struct{}{}
// default:
// }
// }
//}
//var (
// clusterIDs []string
// nodeIDs []string
// clusterIDToNames = map[string]string{}
// nodeIDToNames = map[string]string{}
//)
//if len(clusterIDsM) > 0 {
// for k, _ := range clusterIDsM {
// clusterIDs = append(clusterIDs, k)
// }
// clusterIDToNames, err = common2.GetClusterNames(clusterIDs)
// if err != nil {
// return nil,queryResult, err
// }
//}
//if len(nodeIDsM) > 0 {
// for k, _ := range nodeIDsM {
// nodeIDs = append(nodeIDs, k)
// }
// nodeIDToNames, err = common2.GetNodeNames(nodeIDs)
// if err != nil {
// return nil,queryResult, err
// }
//}
for _, md := range metricData {
if len(md.Data) == 0 {
@ -863,30 +865,31 @@ func getRuleMetricData( rule *alerting.Rule, filterParam *alerting.FilterParam)
}
}
displayGroupValues := make([]string, len(md.GroupValues))
for i, gv := range md.GroupValues {
switch rule.Metrics.Groups[i].Field {
case "metadata.labels.cluster_id", "metadata.cluster_id":
if name, ok := clusterIDToNames[gv]; ok && name != "" {
displayGroupValues[i] = name
}else{
displayGroupValues[i] = gv
}
case "metadata.node_id", "metadata.labels.node_id":
if name, ok := nodeIDToNames[gv]; ok && name != "" {
displayGroupValues[i] = name
}else{
displayGroupValues[i] = gv
}
default:
displayGroupValues[i] = gv
}
}
//displayGroupValues := make([]string, len(md.GroupValues))
//for i, gv := range md.GroupValues {
// switch rule.Metrics.Groups[i].Field {
// case "metadata.labels.cluster_id", "metadata.cluster_id":
// if name, ok := clusterIDToNames[gv]; ok && name != "" {
// displayGroupValues[i] = name
// }else{
// displayGroupValues[i] = gv
// }
// case "metadata.node_id", "metadata.labels.node_id":
// if name, ok := nodeIDToNames[gv]; ok && name != "" {
// displayGroupValues[i] = name
// }else{
// displayGroupValues[i] = gv
// }
// default:
// displayGroupValues[i] = gv
// }
//}
var label = strings.Join(displayGroupValues, "-")
var label = strings.Join(md.GroupValues, "-")
if label == "" {
label, _ = rule.GetOrInitExpression()
}
metricItem.BucketGroups = append(metricItem.BucketGroups, md.GroupValues)
metricItem.Lines = append(metricItem.Lines, &common.MetricLine{
Data: targetData,
BucketSize: filterParam.BucketSize,
@ -899,4 +902,4 @@ func getRuleMetricData( rule *alerting.Rule, filterParam *alerting.FilterParam)
})
}
return &metricItem,queryResult, nil
}
}

40
plugin/api/insight/map_label.go
View File

@ -5,6 +5,7 @@
package insight
import (
"fmt"
log "github.com/cihub/seelog"
common2 "infini.sh/console/common"
httprouter "infini.sh/framework/core/api/router"
@ -28,9 +29,39 @@ func (h *InsightAPI) renderMapLabelTemplate(w http.ResponseWriter, req *http.Req
h.WriteError(w, "bad request", http.StatusInternalServerError)
}
client := elastic.GetClient(clusterID)
var isFakeRender = true
cacheLabelsMap := map[string]map[string]string{}
keyFieldValuesM := map[string]map[string]struct{}{}
tpl, err := template.New("template_render").Funcs(map[string]any{
"map_label": func(indexName, indexKeyField, indexValueField, labelName string) string {
return common2.MapLabel(labelName, indexName, indexKeyField, indexValueField, client)
"map_label": func(indexName, keyField, valueField, labelName string) string {
cacheKey := fmt.Sprintf("%s_%s_%s", indexName, keyField, valueField)
if isFakeRender {
if keyFieldValuesM[cacheKey] == nil {
keyFieldValuesM[cacheKey] = map[string]struct{}{}
}
keyFieldValuesM[cacheKey][labelName] = struct{}{}
return ""
}
var (
cacheLabels map[string]string
ok bool
)
if cacheLabels, ok = cacheLabelsMap[cacheKey]; !ok {
var keyFieldValues []string
if v, ok := keyFieldValuesM[cacheKey]; ok {
keyFieldValues = make([]string, 0, len(v))
for key, _ := range v {
keyFieldValues = append(keyFieldValues, key)
}
}
cacheLabels, err = common2.GetLabelMaps(indexName, keyField, valueField, client, keyFieldValues, len(keyFieldValues))
if err != nil {
log.Error(err)
}else{
cacheLabelsMap[cacheKey] = cacheLabels
}
}
return common2.MapLabel(labelName, indexName, keyField, valueField, client, cacheLabels)
},
}).Parse(body.Template)
if err != nil {
@ -38,6 +69,11 @@ func (h *InsightAPI) renderMapLabelTemplate(w http.ResponseWriter, req *http.Req
h.WriteError(w, err.Error(), http.StatusInternalServerError)
return
}
//do fake render
for _, ctx := range body.Contexts {
common2.ExecuteTemplate(tpl, ctx.Value)
}
isFakeRender = false
resultLabels := map[string]string{}
for _, ctx := range body.Contexts {
label, err := common2.ExecuteTemplate(tpl, ctx.Value)