floraachy
/
console
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[plugin][api] visualization/data supports top_hits

This commit is contained in:
Kassian Sun 2023-03-22 10:10:30 +08:00
parent 157fc43ee8
commit 17abd00e20
1 changed files with 104 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
plugin/api/insight/metric_util.go
View File

@ -6,14 +6,15 @@ package insight
import (
"fmt"
"strconv"
"strings"
"infini.sh/framework/core/elastic"
"infini.sh/framework/core/insight"
"infini.sh/framework/core/util"
"strconv"
"strings"
)
func generateAgg(metricItem *insight.MetricItem) map[string]interface{}{
func generateAgg(metricItem *insight.MetricItem, timeField string) map[string]interface{} {
var (
aggType = "value_count"
field = metricItem.Field
@ -33,13 +34,28 @@ func generateAgg(metricItem *insight.MetricItem) map[string]interface{}{
isPipeline = true
case "medium": // from es version 6.6
aggType = "median_absolute_deviation"
case "p99", "p95","p90","p80","p50":
case "p99", "p95", "p90", "p80", "p50":
aggType = "percentiles"
percentStr := strings.TrimPrefix(metricItem.Statistic, "p")
percent, _ = strconv.ParseFloat(percentStr, 32)
case "latest":
aggType = "top_hits"
}
aggValue := util.MapStr{
"field": field,
aggValue := util.MapStr{}
if aggType != "top_hits" {
aggValue["field"] = field
} else {
aggValue["_source"] = util.MapStr{
"includes": []string{field},
}
aggValue["sort"] = []util.MapStr{
util.MapStr{
timeField: util.MapStr{
"order": "desc",
},
},
}
aggValue["size"] = 1
}
if aggType == "percentiles" {
aggValue["percents"] = []interface{}{percent}
@ -49,7 +65,7 @@ func generateAgg(metricItem *insight.MetricItem) map[string]interface{}{
aggType: aggValue,
},
}
if !isPipeline{
if !isPipeline {
return aggs
}
pipelineAggID := util.GetUUID()
@ -70,14 +86,14 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) {
if metricItem.Name == "" {
metricItem.Name = string('a' + i)
}
metricAggs := generateAgg(&metricItem)
metricAggs := generateAgg(&metricItem, metric.TimeField)
if err := util.MergeFields(basicAggs, metricAggs, true); err != nil {
return nil, err
}
}
verInfo := elastic.GetClient(metric.ClusterId).GetVersion()
if verInfo.Number==""{
if verInfo.Number == "" {
panic("invalid version")
}
@ -85,7 +101,7 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) {
if err != nil {
return nil, fmt.Errorf("get interval field error: %w", err)
}
if metric.TimeField != "" && !timeBeforeGroup{
if metric.BucketSize != "" && !timeBeforeGroup {
basicAggs = util.MapStr{
"time_buckets": util.MapStr{
"date_histogram": util.MapStr{
@ -97,14 +113,13 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) {
}
}
var rootAggs util.MapStr
groups := metric.Groups
if grpLength := len(groups); grpLength > 0 {
var lastGroupAgg util.MapStr
for i := grpLength-1; i>=0; i-- {
for i := grpLength - 1; i >= 0; i-- {
limit := groups[i].Limit
//top group 10
if limit <= 0 {
@ -121,16 +136,16 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) {
groupAgg["aggs"] = util.MapStr{
groupID: lastGroupAgg,
}
}else{
} else {
groupAgg["aggs"] = basicAggs
}
lastGroupAgg = groupAgg
}
if metric.TimeField == "" || (metric.TimeField != "" && !timeBeforeGroup) {
if metric.BucketSize == "" || (metric.BucketSize != "" && !timeBeforeGroup) {
rootAggs = util.MapStr{
util.GetUUID(): lastGroupAgg,
}
}else{
} else {
rootAggs = util.MapStr{
"time_buckets": util.MapStr{
"date_histogram": util.MapStr{
@ -144,8 +159,8 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) {
}
}
}else{
if metric.TimeField != "" && timeBeforeGroup{
} else {
if metric.BucketSize != "" && timeBeforeGroup {
basicAggs = util.MapStr{
"time_buckets": util.MapStr{
"date_histogram": util.MapStr{
@ -178,18 +193,18 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) {
return queryDsl, nil
}
func CollectMetricData(agg interface{}, timeBeforeGroup bool) []insight.MetricData{
func CollectMetricData(agg interface{}, timeBeforeGroup bool) []insight.MetricData {
metricData := []insight.MetricData{}
if timeBeforeGroup {
collectMetricDataOther(agg, "", &metricData, nil)
}else{
} else {
collectMetricData(agg, "", &metricData)
}
return metricData
}
//timeBeforeGroup => false
func collectMetricData(agg interface{}, groupValues string, metricData *[]insight.MetricData){
// timeBeforeGroup => false
func collectMetricData(agg interface{}, groupValues string, metricData *[]insight.MetricData) {
if aggM, ok := agg.(map[string]interface{}); ok {
if timeBks, ok := aggM["time_buckets"].(map[string]interface{}); ok {
if bks, ok := timeBks["buckets"].([]interface{}); ok {
@ -198,14 +213,14 @@ func collectMetricData(agg interface{}, groupValues string, metricData *[]insigh
Group: groupValues,
}
for _, bk := range bks {
if bkM, ok := bk.(map[string]interface{}); ok{
if bkM, ok := bk.(map[string]interface{}); ok {
for k, v := range bkM {
if k == "key" || k == "key_as_string" || k== "doc_count"{
if k == "key" || k == "key_as_string" || k == "doc_count" {
continue
}
if vm, ok := v.(map[string]interface{}); ok && len(k) < 5{
if vm, ok := v.(map[string]interface{}); ok && len(k) < 5 {
collectMetricDataItem(k, vm, &md, bkM["key"])
}
@ -213,16 +228,16 @@ func collectMetricData(agg interface{}, groupValues string, metricData *[]insigh
}
}
*metricData = append(*metricData,md)
*metricData = append(*metricData, md)
}
}else{
} else {
md := insight.MetricData{
Data: map[string][]insight.MetricDataItem{},
Group: groupValues,
}
for k, v := range aggM {
if k == "key" || k== "doc_count"{
if k == "key" || k == "doc_count" {
continue
}
if vm, ok := v.(map[string]interface{}); ok {
@ -237,7 +252,7 @@ func collectMetricData(agg interface{}, groupValues string, metricData *[]insigh
collectMetricData(bk, newGroupValues, metricData)
}
}
}else{
} else {
//non time series metric data
if len(k) < 5 {
collectMetricDataItem(k, vm, &md, nil)
@ -246,14 +261,14 @@ func collectMetricData(agg interface{}, groupValues string, metricData *[]insigh
}
}
if len(md.Data) > 0 {
*metricData = append(*metricData,md)
*metricData = append(*metricData, md)
}
}
}
}
//timeBeforeGroup => true
func collectMetricDataOther(agg interface{}, groupValues string, metricData *[]insight.MetricData, timeKey interface{}){
// timeBeforeGroup => true
func collectMetricDataOther(agg interface{}, groupValues string, metricData *[]insight.MetricData, timeKey interface{}) {
if aggM, ok := agg.(map[string]interface{}); ok {
if timeBks, ok := aggM["time_buckets"].(map[string]interface{}); ok {
if bks, ok := timeBks["buckets"].([]interface{}); ok {
@ -262,15 +277,15 @@ func collectMetricDataOther(agg interface{}, groupValues string, metricData *[]i
Group: groupValues,
}
for _, bk := range bks {
if bkM, ok := bk.(map[string]interface{}); ok{
if bkM, ok := bk.(map[string]interface{}); ok {
for k, v := range bkM {
if k == "key" || k == "key_as_string" || k== "doc_count"{
if k == "key" || k == "key_as_string" || k == "doc_count" {
continue
}
if vm, ok := v.(map[string]interface{}); ok {
if vm["buckets"] != nil {
collectMetricDataOther(vm, groupValues, metricData, bkM["key"])
}else{
} else {
collectMetricDataItem(k, vm, &md, bkM["key"])
}
}
@ -278,10 +293,10 @@ func collectMetricDataOther(agg interface{}, groupValues string, metricData *[]i
}
}
if len(md.Data) > 0 {
*metricData = append(*metricData,md)
*metricData = append(*metricData, md)
}
}
}else{
} else {
md := insight.MetricData{
Data: map[string][]insight.MetricDataItem{},
Group: groupValues,
@ -298,13 +313,13 @@ func collectMetricDataOther(agg interface{}, groupValues string, metricData *[]i
collectMetricDataOther(bk, newGroupValues, metricData, timeKey)
}
}
}else{
} else {
//non time series metric data
for k, v := range aggM {
if vm, ok := v.(map[string]interface{}); ok {
if vm["buckets"] != nil {
collectMetricDataOther(vm, groupValues, metricData, timeKey)
}else{
} else {
collectMetricDataItem(k, vm, &md, timeKey)
}
}
@ -312,19 +327,34 @@ func collectMetricDataOther(agg interface{}, groupValues string, metricData *[]i
}
if len(md.Data) > 0 {
*metricData = append(*metricData,md)
*metricData = append(*metricData, md)
}
}
}
}
func collectMetricDataItem(key string, vm map[string]interface{}, metricData *insight.MetricData, timeKey interface{}) {
if val, ok := vm["value"]; ok {
metricData.Data[key] = append(metricData.Data[key], insight.MetricDataItem{
Value: val,
Timestamp: timeKey,
})
} else if hits, ok := vm["hits"]; ok {
if hits, ok := hits.(map[string]interface{}); ok {
// statistic: top_hits
if hits, ok := hits["hits"]; ok {
if hits, ok := hits.([]interface{}); ok {
for _, hit := range hits {
if hit, ok := hit.(map[string]interface{}); ok {
metricData.Data[key] = append(metricData.Data[key], insight.MetricDataItem{
Value: extractSomeValue(hit["_source"]),
Timestamp: timeKey,
})
}
}
}
}
}
} else {
//percentiles agg type
switch vm["values"].(type) {
@ -350,3 +380,11 @@ func collectMetricDataItem(key string, vm map[string]interface{}, metricData *in
}
}
func extractSomeValue(v interface{}) interface{} {
if vm, ok := v.(map[string]interface{}); ok {
for _, v := range vm {
return extractSomeValue(v)
}
}
return v
}