floraachy
/
console
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

optimize map label api

This commit is contained in:
liugq 2023-06-19 07:21:22 +08:00
parent 68482f5fec
commit 1fe5ace58d
4 changed files with 139 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

132
common/elastic.go
View File

@ -9,81 +9,10 @@ import (
"fmt" "fmt"
log "github.com/cihub/seelog" log "github.com/cihub/seelog"
"infini.sh/framework/core/elastic" "infini.sh/framework/core/elastic"
"infini.sh/framework/core/orm"
"infini.sh/framework/core/util" "infini.sh/framework/core/util"
"sync"
"text/template" "text/template"
"time"
) )
//GetClusterNames query cluster names by cluster ids
func GetClusterNames(clusterIds []string) (map[string]string, error){
if len(clusterIds) == 0 {
return nil, fmt.Errorf("cluster ids must not be empty")
}
rq := util.MapStr{
"size": len(clusterIds),
"query": util.MapStr{
"terms": util.MapStr{
"id": clusterIds,
},
},
}
idToNames := map[string]string{}
q := orm.Query{
RawQuery: util.MustToJSONBytes(rq),
}
err, result := orm.Search(elastic.ElasticsearchConfig{}, &q)
if err != nil {
return nil, err
}
for _, row := range result.Result {
if rowM, ok := row.(map[string]interface{}); ok {
if id, ok := rowM["id"].(string); ok {
if name, ok := rowM["name"].(string); ok {
idToNames[id] = name
}
}
}
}
return idToNames, nil
}
//GetNodeNames query node names by node ids
func GetNodeNames(nodeIDs []string) (map[string]string, error){
if len(nodeIDs) == 0 {
return nil, fmt.Errorf("node ids must not be empty")
}
rq := util.MapStr{
"size": 500,
"query": util.MapStr{
"terms": util.MapStr{
"metadata.node_id": nodeIDs,
},
},
}
idToNames := map[string]string{}
q := orm.Query{
RawQuery: util.MustToJSONBytes(rq),
}
err, result := orm.Search(elastic.NodeConfig{}, &q)
if err != nil {
return nil, err
}
for _, row := range result.Result {
if rowM, ok := row.(map[string]interface{}); ok {
id := GetMapStringValue(rowM, "metadata.node_id")
name := GetMapStringValue(rowM, "metadata.node_name")
if id != "" {
idToNames[id] = name
}
}
}
return idToNames, nil
}
func GetMapStringValue(m util.MapStr, key string) string { func GetMapStringValue(m util.MapStr, key string) string {
v, err := m.GetValue(key) v, err := m.GetValue(key)
if err != nil { if err != nil {
@ -92,8 +21,13 @@ func GetMapStringValue(m util.MapStr, key string) string {
return util.ToString(v) return util.ToString(v)
} }
func MapLabel(labelName, indexName, indexKeyField, indexValueField string, client elastic.API) string { func MapLabel(labelName, indexName, keyField, valueField string, client elastic.API, cacheLabels map[string]string) string {
labelMaps, err := getOrInitLabelCache(indexName, indexKeyField, indexValueField, client) if len(cacheLabels) > 0 {
if v, ok := cacheLabels[labelName]; ok{
return v
}
}
labelMaps, err := GetLabelMaps(indexName, keyField, valueField, client, []string{labelName}, 1)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
return "" return ""
@ -101,46 +35,24 @@ func MapLabel(labelName, indexName, indexKeyField, indexValueField string, clien
return labelMaps[labelName] return labelMaps[labelName]
} }
var labelCache = sync.Map{} func GetLabelMaps( indexName, keyField, valueField string, client elastic.API, keyFieldValues []string, cacheSize int) (map[string]string, error){
type LabelCacheItem struct {
KeyValues map[string]string
Timestamp time.Time
}
func getOrInitLabelCache(indexName, indexKeyField, indexValueField string, client elastic.API) (map[string]string, error){
cacheKey := fmt.Sprintf("%s_%s_%s", indexName, indexKeyField, indexValueField )
var (
labelMaps = map[string]string{}
err error
)
if v, ok := labelCache.Load(cacheKey); ok {
if cacheItem, ok := v.(*LabelCacheItem); ok {
if cacheItem.Timestamp.Add(time.Minute).After(time.Now()){
return cacheItem.KeyValues, nil
}
//cache expired
}
}
labelMaps, err = getLabelMaps(indexName, indexKeyField, indexValueField, client)
if err != nil {
return labelMaps, err
}
labelCache.Store(cacheKey, &LabelCacheItem{
KeyValues: labelMaps,
Timestamp: time.Now(),
})
return labelMaps, nil
}
func getLabelMaps( indexName, indexKeyField, indexValueField string, client elastic.API) (map[string]string, error){
if client == nil { if client == nil {
return nil, fmt.Errorf("cluster client must not be empty") return nil, fmt.Errorf("cluster client must not be empty")
} }
query := util.MapStr{ query := util.MapStr{
"size": 1000, "size": cacheSize,
"collapse": util.MapStr{ "collapse": util.MapStr{
"field": indexKeyField, "field": keyField,
}, },
"_source": []string{indexKeyField, indexValueField}, "_source": []string{keyField, valueField},
}
if len(keyFieldValues) > 0 {
query["query"] = util.MapStr{
"terms": util.MapStr{
keyField: keyFieldValues,
},
}
query["size"] = len(keyFieldValues)
} }
queryDsl := util.MustToJSONBytes(query) queryDsl := util.MustToJSONBytes(query)
searchRes, err := client.SearchWithRawQueryDSL(indexName, queryDsl) searchRes, err := client.SearchWithRawQueryDSL(indexName, queryDsl)
@ -150,12 +62,12 @@ func getLabelMaps( indexName, indexKeyField, indexValueField string, client elas
labelMaps := map[string]string{} labelMaps := map[string]string{}
for _, hit := range searchRes.Hits.Hits { for _, hit := range searchRes.Hits.Hits {
sourceM := util.MapStr(hit.Source) sourceM := util.MapStr(hit.Source)
v := GetMapStringValue(sourceM, indexValueField) v := GetMapStringValue(sourceM, valueField)
var key string var key string
if indexKeyField == "_id" { if keyField == "_id" {
key = hit.ID key = hit.ID
}else{ }else{
key = GetMapStringValue(sourceM, indexKeyField) key = GetMapStringValue(sourceM, keyField)
} }
if key != "" { if key != "" {
labelMaps[key] = v labelMaps[key] = v

6
model/alerting/metric.go
View File

@ -7,6 +7,7 @@ package alerting
import ( import (
"fmt" "fmt"
"infini.sh/console/model/insight" "infini.sh/console/model/insight"
"infini.sh/framework/modules/elastic/common"
"regexp" "regexp"
) )
@ -60,3 +61,8 @@ type MetricData struct {
} }
type TimeMetricData []interface{} type TimeMetricData []interface{}
type AlertMetricItem struct {
common.MetricItem
BucketGroups [][]string `json:"bucket_groups,omitempty"`
}

127
plugin/api/alerting/rule.go
View File

@ -9,7 +9,6 @@ import (
"fmt" "fmt"
log "github.com/cihub/seelog" log "github.com/cihub/seelog"
"github.com/r3labs/diff/v2" "github.com/r3labs/diff/v2"
common2 "infini.sh/console/common"
"infini.sh/console/model/alerting" "infini.sh/console/model/alerting"
"infini.sh/console/model/insight" "infini.sh/console/model/insight"
alerting2 "infini.sh/console/service/alerting" alerting2 "infini.sh/console/service/alerting"
@ -784,6 +783,7 @@ func (alertAPI *AlertAPI) getMetricData(w http.ResponseWriter, req *http.Request
} }
resBody := util.MapStr{ resBody := util.MapStr{
"metric": metricItem, "metric": metricItem,
"bucket_label": rule.Metrics.BucketLabel,
} }
if alertAPI.GetParameter(req, "debug") == "1" { if alertAPI.GetParameter(req, "debug") == "1" {
resBody["query"] = queryResult.Query resBody["query"] = queryResult.Query
@ -791,7 +791,7 @@ func (alertAPI *AlertAPI) getMetricData(w http.ResponseWriter, req *http.Request
alertAPI.WriteJSON(w,resBody, http.StatusOK) alertAPI.WriteJSON(w,resBody, http.StatusOK)
} }
func getRuleMetricData( rule *alerting.Rule, filterParam *alerting.FilterParam) (*common.MetricItem, *alerting.QueryResult, error) { func getRuleMetricData( rule *alerting.Rule, filterParam *alerting.FilterParam) (*alerting.AlertMetricItem, *alerting.QueryResult, error) {
eng := alerting2.GetEngine(rule.Resource.Type) eng := alerting2.GetEngine(rule.Resource.Type)
metricData, queryResult, err := eng.GetTargetMetricData(rule, true, filterParam) metricData, queryResult, err := eng.GetTargetMetricData(rule, true, filterParam)
if err != nil { if err != nil {
@ -802,7 +802,8 @@ func getRuleMetricData( rule *alerting.Rule, filterParam *alerting.FilterParam)
if rule.Metrics.FormatType != "" { if rule.Metrics.FormatType != "" {
formatType = rule.Metrics.FormatType formatType = rule.Metrics.FormatType
} }
var metricItem = common.MetricItem{ var metricItem = alerting.AlertMetricItem{
MetricItem: common.MetricItem{
Group: rule.ID, Group: rule.ID,
Key: rule.ID, Key: rule.ID,
Axis: []*common.MetricAxis{ Axis: []*common.MetricAxis{
@ -810,46 +811,47 @@ func getRuleMetricData( rule *alerting.Rule, filterParam *alerting.FilterParam)
TickFormat: "0,0.[00]", TickFormat: "0,0.[00]",
Ticks: 5}, Ticks: 5},
}, },
},
} }
var ( //var (
clusterIDsM = map[string]struct{}{} // clusterIDsM = map[string]struct{}{}
nodeIDsM = map[string]struct{}{} // nodeIDsM = map[string]struct{}{}
) //)
for _, md := range metricData { //for _, md := range metricData {
for i, gv := range md.GroupValues { // for i, gv := range md.GroupValues {
switch rule.Metrics.Groups[i].Field { // switch rule.Metrics.Groups[i].Field {
case "metadata.labels.cluster_id", "metadata.cluster_id": // case "metadata.labels.cluster_id", "metadata.cluster_id":
clusterIDsM[gv] = struct{}{} // clusterIDsM[gv] = struct{}{}
case "metadata.node_id", "metadata.labels.node_id": // case "metadata.node_id", "metadata.labels.node_id":
nodeIDsM[gv] = struct{}{} // nodeIDsM[gv] = struct{}{}
default: // default:
} // }
} // }
} //}
var ( //var (
clusterIDs []string // clusterIDs []string
nodeIDs []string // nodeIDs []string
clusterIDToNames = map[string]string{} // clusterIDToNames = map[string]string{}
nodeIDToNames = map[string]string{} // nodeIDToNames = map[string]string{}
) //)
if len(clusterIDsM) > 0 { //if len(clusterIDsM) > 0 {
for k, _ := range clusterIDsM { // for k, _ := range clusterIDsM {
clusterIDs = append(clusterIDs, k) // clusterIDs = append(clusterIDs, k)
} // }
clusterIDToNames, err = common2.GetClusterNames(clusterIDs) // clusterIDToNames, err = common2.GetClusterNames(clusterIDs)
if err != nil { // if err != nil {
return nil,queryResult, err // return nil,queryResult, err
} // }
} //}
if len(nodeIDsM) > 0 { //if len(nodeIDsM) > 0 {
for k, _ := range nodeIDsM { // for k, _ := range nodeIDsM {
nodeIDs = append(nodeIDs, k) // nodeIDs = append(nodeIDs, k)
} // }
nodeIDToNames, err = common2.GetNodeNames(nodeIDs) // nodeIDToNames, err = common2.GetNodeNames(nodeIDs)
if err != nil { // if err != nil {
return nil,queryResult, err // return nil,queryResult, err
} // }
} //}
for _, md := range metricData { for _, md := range metricData {
if len(md.Data) == 0 { if len(md.Data) == 0 {
@ -863,30 +865,31 @@ func getRuleMetricData( rule *alerting.Rule, filterParam *alerting.FilterParam)
} }
} }
displayGroupValues := make([]string, len(md.GroupValues)) //displayGroupValues := make([]string, len(md.GroupValues))
for i, gv := range md.GroupValues { //for i, gv := range md.GroupValues {
switch rule.Metrics.Groups[i].Field { // switch rule.Metrics.Groups[i].Field {
case "metadata.labels.cluster_id", "metadata.cluster_id": // case "metadata.labels.cluster_id", "metadata.cluster_id":
if name, ok := clusterIDToNames[gv]; ok && name != "" { // if name, ok := clusterIDToNames[gv]; ok && name != "" {
displayGroupValues[i] = name // displayGroupValues[i] = name
}else{ // }else{
displayGroupValues[i] = gv // displayGroupValues[i] = gv
} // }
case "metadata.node_id", "metadata.labels.node_id": // case "metadata.node_id", "metadata.labels.node_id":
if name, ok := nodeIDToNames[gv]; ok && name != "" { // if name, ok := nodeIDToNames[gv]; ok && name != "" {
displayGroupValues[i] = name // displayGroupValues[i] = name
}else{ // }else{
displayGroupValues[i] = gv // displayGroupValues[i] = gv
} // }
default: // default:
displayGroupValues[i] = gv // displayGroupValues[i] = gv
} // }
} //}
var label = strings.Join(displayGroupValues, "-") var label = strings.Join(md.GroupValues, "-")
if label == "" { if label == "" {
label, _ = rule.GetOrInitExpression() label, _ = rule.GetOrInitExpression()
} }
metricItem.BucketGroups = append(metricItem.BucketGroups, md.GroupValues)
metricItem.Lines = append(metricItem.Lines, &common.MetricLine{ metricItem.Lines = append(metricItem.Lines, &common.MetricLine{
Data: targetData, Data: targetData,
BucketSize: filterParam.BucketSize, BucketSize: filterParam.BucketSize,

40
plugin/api/insight/map_label.go
View File

@ -5,6 +5,7 @@
package insight package insight
import ( import (
"fmt"
log "github.com/cihub/seelog" log "github.com/cihub/seelog"
common2 "infini.sh/console/common" common2 "infini.sh/console/common"
httprouter "infini.sh/framework/core/api/router" httprouter "infini.sh/framework/core/api/router"
@ -28,9 +29,39 @@ func (h *InsightAPI) renderMapLabelTemplate(w http.ResponseWriter, req *http.Req
h.WriteError(w, "bad request", http.StatusInternalServerError) h.WriteError(w, "bad request", http.StatusInternalServerError)
} }
client := elastic.GetClient(clusterID) client := elastic.GetClient(clusterID)
var isFakeRender = true
cacheLabelsMap := map[string]map[string]string{}
keyFieldValuesM := map[string]map[string]struct{}{}
tpl, err := template.New("template_render").Funcs(map[string]any{ tpl, err := template.New("template_render").Funcs(map[string]any{
"map_label": func(indexName, indexKeyField, indexValueField, labelName string) string { "map_label": func(indexName, keyField, valueField, labelName string) string {
return common2.MapLabel(labelName, indexName, indexKeyField, indexValueField, client) cacheKey := fmt.Sprintf("%s_%s_%s", indexName, keyField, valueField)
if isFakeRender {
if keyFieldValuesM[cacheKey] == nil {
keyFieldValuesM[cacheKey] = map[string]struct{}{}
}
keyFieldValuesM[cacheKey][labelName] = struct{}{}
return ""
}
var (
cacheLabels map[string]string
ok bool
)
if cacheLabels, ok = cacheLabelsMap[cacheKey]; !ok {
var keyFieldValues []string
if v, ok := keyFieldValuesM[cacheKey]; ok {
keyFieldValues = make([]string, 0, len(v))
for key, _ := range v {
keyFieldValues = append(keyFieldValues, key)
}
}
cacheLabels, err = common2.GetLabelMaps(indexName, keyField, valueField, client, keyFieldValues, len(keyFieldValues))
if err != nil {
log.Error(err)
}else{
cacheLabelsMap[cacheKey] = cacheLabels
}
}
return common2.MapLabel(labelName, indexName, keyField, valueField, client, cacheLabels)
}, },
}).Parse(body.Template) }).Parse(body.Template)
if err != nil { if err != nil {
@ -38,6 +69,11 @@ func (h *InsightAPI) renderMapLabelTemplate(w http.ResponseWriter, req *http.Req
h.WriteError(w, err.Error(), http.StatusInternalServerError) h.WriteError(w, err.Error(), http.StatusInternalServerError)
return return
} }
//do fake render
for _, ctx := range body.Contexts {
common2.ExecuteTemplate(tpl, ctx.Value)
}
isFakeRender = false
resultLabels := map[string]string{} resultLabels := map[string]string{}
for _, ctx := range body.Contexts { for _, ctx := range body.Contexts {
label, err := common2.ExecuteTemplate(tpl, ctx.Value) label, err := common2.ExecuteTemplate(tpl, ctx.Value)