floraachy
/
console
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: code format (#72) 

* chore: code format

* chore: remove fetch-depth

* chore: add format and lint

* chore: add pr_check

* fix: lint with config

* chore: this pr only unit test

* fix: code format error
This commit is contained in:
Hardy 2025-01-14 14:29:31 +08:00 committed by GitHub
parent fb4dafecb3
commit 8da176bea8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
88 changed files with 3497 additions and 3365 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

307
.github/workflows/pr_check.yml vendored Normal file
View File

@ -0,0 +1,307 @@
name: Unit Test
on:
pull_request:
branches: [ "main" ]
defaults:
run:
shell: bash
env:
GO_VERSION: 1.23.4
NODEJS_VERSION: 16.20.2
PNAME: console
jobs:
format_check:
runs-on: ubuntu-latest
steps:
- name: Checkout current repository
uses: actions/checkout@v4
with:
path: ${{ env.PNAME }}
- name: Checkout framework repository
uses: actions/checkout@v4
with:
repository: infinilabs/framework
path: framework
- name: Checkout framework-vendor
uses: actions/checkout@v4
with:
ref: main
repository: infinilabs/framework-vendor
path: vendor
- name: Set up nodejs toolchain
uses: actions/setup-node@v4
with:
node-version: ${{ env.NODEJS_VERSION }}
- name: Cache dependencies
uses: actions/cache@v4
with:
path: |
node_modules
key: ${{ runner.os }}-cnpm-${{ hashFiles('**/package.json') }}
restore-keys: |
${{ runner.os }}-cnpm-
- name: Check nodejs toolchain
run: |
if ! command -v cnpm >/dev/null 2>&1; then
npm install -g rimraf --quiet --no-progress
npm install -g cnpm@9.2.0 --quiet --no-progress
fi
node -v && npm -v && cnpm -v
- name: Set up go toolchain
uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
check-latest: false
cache: true
- name: Check go toolchain
run: go version
- name: Cache Build Output
uses: actions/cache@v4
with:
path: |
.public
key: ${{ runner.os }}-build-${{ hashFiles('**/package.json') }}-${{ github.sha }}
restore-keys: |
${{ runner.os }}-build-${{ hashFiles('**/package.json') }}-
${{ runner.os }}-build-
- name: Code format
env:
GOFLAGS: -tags=ci
run: |
echo Home path is $HOME
export WORKBASE=$HOME/go/src/infini.sh
export WORK=$WORKBASE/console
# for test workspace
mkdir -p $HOME/go/src/
ln -s $GITHUB_WORKSPACE $WORKBASE
# for web build
cd $WORK/web
cnpm install --quiet --no-progress
cnpm run build --quiet
# check work folder
ls -lrt $WORKBASE/
ls -alrt $WORK
# for code format
cd $WORK
echo Formating code at $PWD ...
make format
if [ $? -ne 0 ]; then
echo "make format failed, please check make output"
exit 1
fi
- name: Check for changes after format
id: check-changes
shell: bash
run: |
export WORKBASE=$HOME/go/src/infini.sh
export WORK=$WORKBASE/$PNAME
# for foramt check
cd $WORK
if [[ $(git status --porcelain | grep -c " M .*\.go$") -gt 0 ]]; then
echo "go format detected formatting changes"
echo "changes=true" >> $GITHUB_OUTPUT
else
echo "go format no changes found"
echo "changes=false" >> $GITHUB_OUTPUT
fi
- name: Fail workflow if changes after format
if: steps.check-changes.outputs.changes == 'true'
run: exit 1
unit_test:
runs-on: ubuntu-latest
steps:
- name: Checkout current repository
uses: actions/checkout@v4
with:
path: ${{ env.PNAME }}
- name: Checkout framework repository
uses: actions/checkout@v4
with:
repository: infinilabs/framework
path: framework
- name: Checkout framework-vendor
uses: actions/checkout@v4
with:
ref: main
repository: infinilabs/framework-vendor
path: vendor
- name: Set up nodejs toolchain
uses: actions/setup-node@v4
with:
node-version: ${{ env.NODEJS_VERSION }}
- name: Cache dependencies
uses: actions/cache@v4
with:
path: |
node_modules
key: ${{ runner.os }}-cnpm-${{ hashFiles('**/package.json') }}
restore-keys: |
${{ runner.os }}-cnpm-
- name: Check nodejs toolchain
run: |
if ! command -v cnpm >/dev/null 2>&1; then
npm install -g rimraf --quiet --no-progress
npm install -g cnpm@9.2.0 --quiet --no-progress
fi
node -v && npm -v && cnpm -v
- name: Set up go toolchain
uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
check-latest: false
cache: true
- name: Check go toolchain
run: go version
- name: Cache Build Output
uses: actions/cache@v4
with:
path: |
.public
key: ${{ runner.os }}-build-${{ hashFiles('**/package.json') }}-${{ github.sha }}
restore-keys: |
${{ runner.os }}-build-${{ hashFiles('**/package.json') }}-
${{ runner.os }}-build-
- name: Unit test
env:
GOFLAGS: -tags=ci
run: |
echo Home path is $HOME
export WORKBASE=$HOME/go/src/infini.sh
export WORK=$WORKBASE/$PNAME
# for test workspace
mkdir -p $HOME/go/src/
ln -s $GITHUB_WORKSPACE $WORKBASE
# for web build
cd $WORK/web
cnpm install --quiet --no-progress
cnpm run build --quiet
# check work folder
ls -lrt $WORKBASE/
ls -alrt $WORK
# for unit test
cd $WORK
echo Testing code at $PWD ...
make test
code_lint:
runs-on: ubuntu-latest
steps:
- name: Checkout current repository
uses: actions/checkout@v4
with:
path: ${{ env.PNAME }}
- name: Checkout framework repository
uses: actions/checkout@v4
with:
repository: infinilabs/framework
path: framework
- name: Checkout framework-vendor
uses: actions/checkout@v4
with:
ref: main
repository: infinilabs/framework-vendor
path: vendor
- name: Set up nodejs toolchain
uses: actions/setup-node@v4
with:
node-version: ${{ env.NODEJS_VERSION }}
- name: Cache dependencies
uses: actions/cache@v4
with:
path: |
node_modules
key: ${{ runner.os }}-cnpm-${{ hashFiles('**/package.json') }}
restore-keys: |
${{ runner.os }}-cnpm-
- name: Check nodejs toolchain
run: |
if ! command -v cnpm >/dev/null 2>&1; then
npm install -g rimraf --quiet --no-progress
npm install -g cnpm@9.2.0 --quiet --no-progress
fi
node -v && npm -v && cnpm -v
- name: Set up go toolchain
uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
check-latest: false
cache: true
- name: Check go toolchain
run: go version
- name: Cache Build Output
uses: actions/cache@v4
with:
path: |
.public
key: ${{ runner.os }}-build-${{ hashFiles('**/package.json') }}-${{ github.sha }}
restore-keys: |
${{ runner.os }}-build-${{ hashFiles('**/package.json') }}-
${{ runner.os }}-build-
- name: Code lint
env:
GOFLAGS: -tags=ci
run: |
echo Home path is $HOME
export WORKBASE=$HOME/go/src/infini.sh
export WORK=$WORKBASE/$PNAME
# for test workspace
mkdir -p $HOME/go/src/
ln -s $GITHUB_WORKSPACE $WORKBASE
# for web build
cd $WORK/web
cnpm install --quiet --no-progress
cnpm run build --quiet
# check work folder
ls -lrt $WORKBASE/
ls -alrt $WORK
# for code lint
cd $WORK
echo Testing code at $PWD ...
# make lint

105
.github/workflows/unit_test.yml vendored
View File

@ -1,105 +0,0 @@
name: Unit Test
on:
pull_request:
branches: [ "main" ]
defaults:
run:
shell: bash
jobs:
build:
runs-on: ubuntu-latest
env:
GO_VERSION: 1.23.4
NODEJS_VERSION: 16.20.2
steps:
- name: Checkout current repository
uses: actions/checkout@v4
with:
fetch-depth: 0
path: console
- name: Checkout framework repository
uses: actions/checkout@v4
with:
fetch-depth: 0
repository: infinilabs/framework
path: framework
- name: Checkout framework-vendor
uses: actions/checkout@v4
with:
ref: main
fetch-depth: 0
repository: infinilabs/framework-vendor
path: vendor
- name: Set up nodejs toolchain
uses: actions/setup-node@v4
with:
node-version: ${{ env.NODEJS_VERSION }}
- name: Cache dependencies
uses: actions/cache@v4
with:
path: |
node_modules
key: ${{ runner.os }}-cnpm-${{ hashFiles('**/package.json') }}
restore-keys: |
${{ runner.os }}-cnpm-
- name: Check nodejs toolchain
run: |
if ! command -v cnpm >/dev/null 2>&1; then
npm install -g rimraf --quiet --no-progress
npm install -g cnpm@9.2.0 --quiet --no-progress
fi
node -v && npm -v && cnpm -v
- name: Set up go toolchain
uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
check-latest: false
cache: true
- name: Check go toolchain
run: go version
- name: Cache Build Output
uses: actions/cache@v4
with:
path: |
.public
key: ${{ runner.os }}-build-${{ hashFiles('**/package.json') }}-${{ github.sha }}
restore-keys: |
${{ runner.os }}-build-${{ hashFiles('**/package.json') }}-
${{ runner.os }}-build-
- name: Unit test
env:
GOFLAGS: -tags=ci
run: |
echo Home path is $HOME
export WORKBASE=$HOME/go/src/infini.sh
export WORK=$WORKBASE/console
# for test workspace
mkdir -p $HOME/go/src/
ln -s $GITHUB_WORKSPACE $WORKBASE
# for web build
cd $WORK/web
cnpm install --quiet --no-progress
cnpm run build --quiet
# check work folder
ls -lrt $WORKBASE/
ls -alrt $WORK
# for unit test
cd $WORK
echo Testing code at $PWD ...
make test

2
.gitignore vendored
View File

@ -32,5 +32,7 @@ appveyor.yml
log/ log/
.env .env
generated_*.go generated_*.go
config/generated.go
config/generat*.go
config/initialization.dsl config/initialization.dsl
config/system_config.yml config/system_config.yml

8
common/elastic.go
View File

@ -46,7 +46,7 @@ func GetMapStringValue(m util.MapStr, key string) string {
func MapLabel(labelName, indexName, keyField, valueField string, client elastic.API, cacheLabels map[string]string) string { func MapLabel(labelName, indexName, keyField, valueField string, client elastic.API, cacheLabels map[string]string) string {
if len(cacheLabels) > 0 { if len(cacheLabels) > 0 {
if v, ok := cacheLabels[labelName]; ok{ if v, ok := cacheLabels[labelName]; ok {
return v return v
} }
} }
@ -58,7 +58,7 @@ func MapLabel(labelName, indexName, keyField, valueField string, client elastic.
return labelMaps[labelName] return labelMaps[labelName]
} }
func GetLabelMaps( indexName, keyField, valueField string, client elastic.API, keyFieldValues []string, cacheSize int) (map[string]string, error){ func GetLabelMaps(indexName, keyField, valueField string, client elastic.API, keyFieldValues []string, cacheSize int) (map[string]string, error) {
if client == nil { if client == nil {
return nil, fmt.Errorf("cluster client must not be empty") return nil, fmt.Errorf("cluster client must not be empty")
} }
@ -89,7 +89,7 @@ func GetLabelMaps( indexName, keyField, valueField string, client elastic.API, k
var key string var key string
if keyField == "_id" { if keyField == "_id" {
key = hit.ID key = hit.ID
}else{ } else {
key = GetMapStringValue(sourceM, keyField) key = GetMapStringValue(sourceM, keyField)
} }
if key != "" { if key != "" {
@ -99,7 +99,7 @@ func GetLabelMaps( indexName, keyField, valueField string, client elastic.API, k
return labelMaps, nil return labelMaps, nil
} }
func ExecuteTemplate( tpl *template.Template, ctx map[string]interface{}) ([]byte, error){ func ExecuteTemplate(tpl *template.Template, ctx map[string]interface{}) ([]byte, error) {
msgBuffer := &bytes.Buffer{} msgBuffer := &bytes.Buffer{}
err := tpl.Execute(msgBuffer, ctx) err := tpl.Execute(msgBuffer, ctx)
return msgBuffer.Bytes(), err return msgBuffer.Bytes(), err

10
config/generated.go
View File

@ -1,10 +0,0 @@
package config
const LastCommitLog = "N/A"
const BuildDate = "N/A"
const EOLDate = "N/A"
const Version = "0.0.1-SNAPSHOT"
const BuildNumber = "001"

8
core/errors/errors.go
View File

@ -54,22 +54,22 @@ func (err Error) Error() string {
return fmt.Sprintf("%s:%v: %v", err.typ, err.field, err.msg) return fmt.Sprintf("%s:%v: %v", err.typ, err.field, err.msg)
} }
//NewAppError returns an application error // NewAppError returns an application error
func NewAppError(msg any) *Error { func NewAppError(msg any) *Error {
return New(ErrTypeApplication, "", msg) return New(ErrTypeApplication, "", msg)
} }
//NewParamsError returns a request params error // NewParamsError returns a request params error
func NewParamsError(field string, msg any) *Error { func NewParamsError(field string, msg any) *Error {
return New(ErrTypeRequestParams, field, msg) return New(ErrTypeRequestParams, field, msg)
} }
//NewAlreadyExistsError returns an already exists error // NewAlreadyExistsError returns an already exists error
func NewAlreadyExistsError(field string, msg any) *Error { func NewAlreadyExistsError(field string, msg any) *Error {
return New(ErrTypeAlreadyExists, field, msg) return New(ErrTypeAlreadyExists, field, msg)
} }
//NewNotExistsError returns a not exists error // NewNotExistsError returns a not exists error
func NewNotExistsError(field string, msg any) *Error { func NewNotExistsError(field string, msg any) *Error {
return New(ErrTypeNotExists, field, msg) return New(ErrTypeNotExists, field, msg)
} }

5
model/alerting/condition.go
View File

@ -33,7 +33,8 @@ type Condition struct {
Operator string `json:"operator"` Operator string `json:"operator"`
Items []ConditionItem `json:"items"` Items []ConditionItem `json:"items"`
} }
func (cond *Condition) GetMinimumPeriodMatch() int{
func (cond *Condition) GetMinimumPeriodMatch() int {
var minPeriodMatch = 0 var minPeriodMatch = 0
for _, citem := range cond.Items { for _, citem := range cond.Items {
if citem.MinimumPeriodMatch > minPeriodMatch { if citem.MinimumPeriodMatch > minPeriodMatch {
@ -52,7 +53,7 @@ type ConditionItem struct {
Expression string `json:"expression,omitempty"` Expression string `json:"expression,omitempty"`
} }
func (cond *ConditionItem) GenerateConditionExpression()(conditionExpression string, err error){ func (cond *ConditionItem) GenerateConditionExpression() (conditionExpression string, err error) {
valueLength := len(cond.Values) valueLength := len(cond.Values)
if valueLength == 0 { if valueLength == 0 {
return conditionExpression, fmt.Errorf("condition values: %v should not be empty", cond.Values) return conditionExpression, fmt.Errorf("condition values: %v should not be empty", cond.Values)

1
model/alerting/destination.go
View File

@ -43,7 +43,6 @@ type Channel struct {
Enabled bool `json:"enabled" elastic_mapping:"enabled:{type:boolean}"` Enabled bool `json:"enabled" elastic_mapping:"enabled:{type:boolean}"`
} }
const ( const (
ChannelEmail = "email" ChannelEmail = "email"
ChannelWebhook = "webhook" ChannelWebhook = "webhook"

5
model/alerting/metric.go
View File

@ -41,8 +41,7 @@ type Metric struct {
Expression string `json:"expression,omitempty" elastic_mapping:"expression:{type:keyword,copy_to:search_text}"` //告警表达式，自动生成 eg: avg(cpu) > 80 Expression string `json:"expression,omitempty" elastic_mapping:"expression:{type:keyword,copy_to:search_text}"` //告警表达式，自动生成 eg: avg(cpu) > 80
} }
func (m *Metric) GenerateExpression() (string, error) {
func (m *Metric) GenerateExpression() (string, error){
if len(m.Items) == 1 { if len(m.Items) == 1 {
return fmt.Sprintf("%s(%s)", m.Items[0].Statistic, m.Items[0].Field), nil return fmt.Sprintf("%s(%s)", m.Items[0].Statistic, m.Items[0].Field), nil
} }
@ -55,7 +54,7 @@ func (m *Metric) GenerateExpression() (string, error){
) )
for _, item := range m.Items { for _, item := range m.Items {
metricExpression = fmt.Sprintf("%s(%s)", item.Statistic, item.Field) metricExpression = fmt.Sprintf("%s(%s)", item.Statistic, item.Field)
reg, err := regexp.Compile(item.Name+`([^\w]|$)`) reg, err := regexp.Compile(item.Name + `([^\w]|$)`)
if err != nil { if err != nil {
return "", err return "", err
} }

3
model/alerting/resource.go
View File

@ -42,10 +42,9 @@ type Resource struct {
Context Context `json:"context"` Context Context `json:"context"`
} }
func (r Resource) Validate() error{ func (r Resource) Validate() error {
if r.TimeField == "" { if r.TimeField == "" {
return fmt.Errorf("TimeField can not be empty") return fmt.Errorf("TimeField can not be empty")
} }
return nil return nil
} }

12
model/alerting/rule.go
View File

@ -58,8 +58,8 @@ type Rule struct {
Tags []string `json:"tags,omitempty" elastic_mapping:"tags: { type: keyword,copy_to:search_text }"` Tags []string `json:"tags,omitempty" elastic_mapping:"tags: { type: keyword,copy_to:search_text }"`
} }
func (rule *Rule) GetOrInitExpression() (string, error){ func (rule *Rule) GetOrInitExpression() (string, error) {
if rule.Expression != ""{ if rule.Expression != "" {
return rule.Expression, nil return rule.Expression, nil
} }
sb := strings.Builder{} sb := strings.Builder{}
@ -81,7 +81,8 @@ func (rule *Rule) GetOrInitExpression() (string, error){
rule.Expression = strings.ReplaceAll(sb.String(), "result", metricExp) rule.Expression = strings.ReplaceAll(sb.String(), "result", metricExp)
return rule.Expression, nil return rule.Expression, nil
} }
//GetNotificationConfig for adapter old version config
// GetNotificationConfig for adapter old version config
func (rule *Rule) GetNotificationConfig() *NotificationConfig { func (rule *Rule) GetNotificationConfig() *NotificationConfig {
if rule.NotificationConfig != nil { if rule.NotificationConfig != nil {
return rule.NotificationConfig return rule.NotificationConfig
@ -116,7 +117,7 @@ type RecoveryNotificationConfig struct {
EventEnabled bool `json:"event_enabled"` EventEnabled bool `json:"event_enabled"`
} }
type MessageTemplate struct{ type MessageTemplate struct {
Type string `json:"type"` Type string `json:"type"`
Source string `json:"source"` Source string `json:"source"`
} }
@ -126,7 +127,7 @@ type TimeRange struct {
End string `json:"end"` End string `json:"end"`
} }
func (tr *TimeRange) Include( t time.Time) bool { func (tr *TimeRange) Include(t time.Time) bool {
if tr.Start == "" || tr.End == "" { if tr.Start == "" || tr.End == "" {
return true return true
} }
@ -139,6 +140,7 @@ type FilterParam struct {
End interface{} `json:"end"` End interface{} `json:"end"`
BucketSize string `json:"bucket_size"` BucketSize string `json:"bucket_size"`
} }
//ctx //ctx
//rule expression, rule_id, resource_id, resource_name, event_id, condition_name, preset_value,[group_tags, check_values], //rule expression, rule_id, resource_id, resource_name, event_id, condition_name, preset_value,[group_tags, check_values],
//check_status ,timestamp, //check_status ,timestamp,

11
model/alerting/rule_test.go
View File

@ -36,7 +36,7 @@ import (
"time" "time"
) )
func TestCreateRule( t *testing.T) { func TestCreateRule(t *testing.T) {
rule := Rule{ rule := Rule{
//ORMObjectBase: orm.ORMObjectBase{ //ORMObjectBase: orm.ORMObjectBase{
// ID: util.GetUUID(), // ID: util.GetUUID(),
@ -87,7 +87,7 @@ func TestCreateRule( t *testing.T) {
Metric: insight.Metric{ Metric: insight.Metric{
Groups: []insight.MetricGroupItem{{"metadata.labels.cluster_id", 10}, {"metadata.labels.node_id", 10}}, Groups: []insight.MetricGroupItem{{"metadata.labels.cluster_id", 10}, {"metadata.labels.node_id", 10}},
Items: []insight.MetricItem{ Items: []insight.MetricItem{
{Name: "a", Field: "payload.elasticsearch.node_stats.fs.total.free_in_bytes", Statistic: "min" }, {Name: "a", Field: "payload.elasticsearch.node_stats.fs.total.free_in_bytes", Statistic: "min"},
{Name: "b", Field: "payload.elasticsearch.node_stats.fs.total.total_in_bytes", Statistic: "max"}, {Name: "b", Field: "payload.elasticsearch.node_stats.fs.total.total_in_bytes", Statistic: "max"},
}, },
BucketSize: "1m", BucketSize: "1m",
@ -145,15 +145,12 @@ func TestCreateRule( t *testing.T) {
fmt.Println(exp) fmt.Println(exp)
} }
func TestTimeRange_Include(t *testing.T) {
func TestTimeRange_Include( t *testing.T) {
tr := TimeRange{ tr := TimeRange{
Start: "08:00", Start: "08:00",
End: "18:31", End: "18:31",
} }
fmt.Println(tr.Include(time.Now())) fmt.Println(tr.Include(time.Now()))
ti,_ := time.Parse(time.RFC3339, "2022-04-11T10:31:38.911000504Z") ti, _ := time.Parse(time.RFC3339, "2022-04-11T10:31:38.911000504Z")
fmt.Println(time.Now().Sub(ti)) fmt.Println(time.Now().Sub(ti))
} }

3
model/alerting/schedule.go
View File

@ -36,6 +36,3 @@ type Cron struct {
Expression string `json:"expression" elastic_mapping:"expression:{type:text}"` Expression string `json:"expression" elastic_mapping:"expression:{type:text}"`
Timezone string `json:"timezone" elastic_mapping:"timezone:{type:keyword}"` Timezone string `json:"timezone" elastic_mapping:"timezone:{type:keyword}"`
} }

1
model/insight/field_meta.go
View File

@ -27,7 +27,6 @@
package insight package insight
type SeriesItem struct { type SeriesItem struct {
Type string `json:"type"` Type string `json:"type"`
Options map[string]interface{} `json:"options"` Options map[string]interface{} `json:"options"`

11
model/insight/metric_data.go
View File

@ -29,9 +29,10 @@ package insight
import ( import (
"fmt" "fmt"
"regexp"
"infini.sh/framework/core/orm" "infini.sh/framework/core/orm"
"infini.sh/framework/core/util" "infini.sh/framework/core/util"
"regexp"
) )
type Metric struct { type Metric struct {
@ -85,7 +86,7 @@ type MetricGroupItem struct {
Limit int `json:"limit"` Limit int `json:"limit"`
} }
func (m *Metric) GenerateExpression() (string, error){ func (m *Metric) GenerateExpression() (string, error) {
if len(m.Items) == 1 { if len(m.Items) == 1 {
return fmt.Sprintf("%s(%s)", m.Items[0].Statistic, m.Items[0].Field), nil return fmt.Sprintf("%s(%s)", m.Items[0].Statistic, m.Items[0].Field), nil
} }
@ -98,7 +99,7 @@ func (m *Metric) GenerateExpression() (string, error){
) )
for _, item := range m.Items { for _, item := range m.Items {
metricExpression = fmt.Sprintf("%s(%s)", item.Statistic, item.Field) metricExpression = fmt.Sprintf("%s(%s)", item.Statistic, item.Field)
reg, err := regexp.Compile(item.Name+`([^\w]|$)`) reg, err := regexp.Compile(item.Name + `([^\w]|$)`)
if err != nil { if err != nil {
return "", err return "", err
} }
@ -127,10 +128,10 @@ func (m *Metric) ValidateSortKey() error {
mm[item.Name] = &item mm[item.Name] = &item
} }
for _, sortItem := range m.Sort { for _, sortItem := range m.Sort {
if !util.StringInArray([]string{"desc", "asc"}, sortItem.Direction){ if !util.StringInArray([]string{"desc", "asc"}, sortItem.Direction) {
return fmt.Errorf("unknown sort direction [%s]", sortItem.Direction) return fmt.Errorf("unknown sort direction [%s]", sortItem.Direction)
} }
if _, ok := mm[sortItem.Key]; !ok && !util.StringInArray([]string{"_key", "_count"}, sortItem.Key){ if _, ok := mm[sortItem.Key]; !ok && !util.StringInArray([]string{"_key", "_count"}, sortItem.Key) {
return fmt.Errorf("unknown sort key [%s]", sortItem.Key) return fmt.Errorf("unknown sort key [%s]", sortItem.Key)
} }
} }

2
model/insight/widget.go
View File

@ -32,5 +32,5 @@ import "infini.sh/framework/core/orm"
type Widget struct { type Widget struct {
orm.ORMObjectBase orm.ORMObjectBase
Title string `json:"title" elastic_mapping:"title: { type: text }"` Title string `json:"title" elastic_mapping:"title: { type: text }"`
Config interface{}`json:"config" elastic_mapping:"config: { type: object,enabled:false }"` Config interface{} `json:"config" elastic_mapping:"config: { type: object,enabled:false }"`
} }

1
model/layout.go
View File

@ -45,6 +45,7 @@ type Layout struct {
} }
type LayoutType string type LayoutType string
const ( const (
LayoutTypeWorkspace LayoutType = "workspace" LayoutTypeWorkspace LayoutType = "workspace"
) )

4
modules/agent/api/host.go
View File

@ -97,7 +97,7 @@ func (h *APIHandler) enrollHost(w http.ResponseWriter, req *http.Request, ps htt
} }
hostInfo.Timestamp = time.Now() hostInfo.Timestamp = time.Now()
var ctx *orm.Context var ctx *orm.Context
if i == len(reqBody) - 1 { if i == len(reqBody)-1 {
ctx = &orm.Context{ ctx = &orm.Context{
Refresh: "wait_for", Refresh: "wait_for",
} }
@ -176,7 +176,7 @@ func (h *APIHandler) GetHostAgentInfo(w http.ResponseWriter, req *http.Request,
}, http.StatusOK) }, http.StatusOK)
} }
func getHost(hostID string) (*host.HostInfo, error){ func getHost(hostID string) (*host.HostInfo, error) {
hostInfo := &host.HostInfo{} hostInfo := &host.HostInfo{}
hostInfo.ID = hostID hostInfo.ID = hostID
exists, err := orm.Get(hostInfo) exists, err := orm.Get(hostInfo)

2
modules/agent/api/remote_config.go
View File

@ -31,13 +31,13 @@ import (
"bytes" "bytes"
"fmt" "fmt"
log "github.com/cihub/seelog" log "github.com/cihub/seelog"
"infini.sh/framework/modules/configs/common"
"infini.sh/framework/core/elastic" "infini.sh/framework/core/elastic"
"infini.sh/framework/core/global" "infini.sh/framework/core/global"
"infini.sh/framework/core/kv" "infini.sh/framework/core/kv"
"infini.sh/framework/core/model" "infini.sh/framework/core/model"
"infini.sh/framework/core/orm" "infini.sh/framework/core/orm"
"infini.sh/framework/core/util" "infini.sh/framework/core/util"
"infini.sh/framework/modules/configs/common"
common2 "infini.sh/framework/modules/elastic/common" common2 "infini.sh/framework/modules/elastic/common"
metadata2 "infini.sh/framework/modules/elastic/metadata" metadata2 "infini.sh/framework/modules/elastic/metadata"
"time" "time"

12
modules/agent/common/cert.go
View File

@ -37,15 +37,15 @@ import (
"path" "path"
) )
func GenerateClientCert(caFile, caKey string) (caCert, clientCertPEM, clientKeyPEM []byte, err error){ func GenerateClientCert(caFile, caKey string) (caCert, clientCertPEM, clientKeyPEM []byte, err error) {
return generateCert(caFile, caKey, false) return generateCert(caFile, caKey, false)
} }
func GenerateServerCert(caFile, caKey string) (caCert, serverCertPEM, serverKeyPEM []byte, err error){ func GenerateServerCert(caFile, caKey string) (caCert, serverCertPEM, serverKeyPEM []byte, err error) {
return generateCert(caFile, caKey, true) return generateCert(caFile, caKey, true)
} }
func generateCert(caFile, caKey string, isServer bool)(caCert, instanceCertPEM, instanceKeyPEM []byte, err error){ func generateCert(caFile, caKey string, isServer bool) (caCert, instanceCertPEM, instanceKeyPEM []byte, err error) {
pool := x509.NewCertPool() pool := x509.NewCertPool()
caCert, err = os.ReadFile(caFile) caCert, err = os.ReadFile(caFile)
if err != nil { if err != nil {
@ -69,11 +69,11 @@ func generateCert(caFile, caKey string, isServer bool)(caCert, instanceCertPEM,
if err != nil { if err != nil {
return return
} }
if isServer{ if isServer {
b = &pem.Block{Type: "CERTIFICATE", Bytes: caCertBytes} b = &pem.Block{Type: "CERTIFICATE", Bytes: caCertBytes}
certPEM := pem.EncodeToMemory(b) certPEM := pem.EncodeToMemory(b)
instanceCertPEM, instanceKeyPEM, err = util.GenerateServerCert(rootCert, certKey.(*rsa.PrivateKey), certPEM, nil) instanceCertPEM, instanceKeyPEM, err = util.GenerateServerCert(rootCert, certKey.(*rsa.PrivateKey), certPEM, nil)
}else{ } else {
_, instanceCertPEM, instanceKeyPEM = util.GetClientCert(rootCert, certKey) _, instanceCertPEM, instanceKeyPEM = util.GetClientCert(rootCert, certKey)
} }
return caCert, instanceCertPEM, instanceKeyPEM, nil return caCert, instanceCertPEM, instanceKeyPEM, nil
@ -96,7 +96,7 @@ func GetAgentInstanceCerts(caFile, caKey string) (string, string, error) {
return "", "", err return "", "", err
} }
baseDir := path.Join(dataDir, "certs/agent") baseDir := path.Join(dataDir, "certs/agent")
if !util.IsExist(baseDir){ if !util.IsExist(baseDir) {
err = os.MkdirAll(baseDir, 0775) err = os.MkdirAll(baseDir, 0775)
if err != nil { if err != nil {
return "", "", err return "", "", err

2
modules/agent/common/config.go
View File

@ -30,8 +30,8 @@ package common
import ( import (
log "github.com/cihub/seelog" log "github.com/cihub/seelog"
"infini.sh/console/modules/agent/model" "infini.sh/console/modules/agent/model"
"infini.sh/framework/modules/configs/common"
"infini.sh/framework/core/env" "infini.sh/framework/core/env"
"infini.sh/framework/modules/configs/common"
) )
func GetAgentConfig() *model.AgentConfig { func GetAgentConfig() *model.AgentConfig {

22
modules/elastic/api/activity.go
View File

@ -38,9 +38,9 @@ import (
"strings" "strings"
) )
func (h *APIHandler) HandleSearchActivityAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ func (h *APIHandler) HandleSearchActivityAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody:=util.MapStr{} resBody := util.MapStr{}
reqBody := struct{ reqBody := struct {
Keyword string `json:"keyword"` Keyword string `json:"keyword"`
Size int `json:"size"` Size int `json:"size"`
From int `json:"from"` From int `json:"from"`
@ -54,7 +54,7 @@ func (h *APIHandler) HandleSearchActivityAction(w http.ResponseWriter, req *http
err := h.DecodeJSON(req, &reqBody) err := h.DecodeJSON(req, &reqBody)
if err != nil { if err != nil {
resBody["error"] = err.Error() resBody["error"] = err.Error()
h.WriteJSON(w,resBody, http.StatusInternalServerError ) h.WriteJSON(w, resBody, http.StatusInternalServerError)
return return
} }
aggs := elastic.BuildSearchTermAggregations(reqBody.Aggregations) aggs := elastic.BuildSearchTermAggregations(reqBody.Aggregations)
@ -86,9 +86,7 @@ func (h *APIHandler) HandleSearchActivityAction(w http.ResponseWriter, req *http
clusterFilter, hasAllPrivilege := h.GetClusterFilter(req, "metadata.labels.cluster_id") clusterFilter, hasAllPrivilege := h.GetClusterFilter(req, "metadata.labels.cluster_id")
if !hasAllPrivilege && clusterFilter == nil { if !hasAllPrivilege && clusterFilter == nil {
h.WriteJSON(w, elastic.SearchResponse{ h.WriteJSON(w, elastic.SearchResponse{}, http.StatusOK)
}, http.StatusOK)
return return
} }
if !hasAllPrivilege && clusterFilter != nil { if !hasAllPrivilege && clusterFilter != nil {
@ -97,9 +95,7 @@ func (h *APIHandler) HandleSearchActivityAction(w http.ResponseWriter, req *http
hasAllPrivilege, indexPrivilege := h.GetCurrentUserIndex(req) hasAllPrivilege, indexPrivilege := h.GetCurrentUserIndex(req)
if !hasAllPrivilege && len(indexPrivilege) == 0 { if !hasAllPrivilege && len(indexPrivilege) == 0 {
h.WriteJSON(w, elastic.SearchResponse{ h.WriteJSON(w, elastic.SearchResponse{}, http.StatusOK)
}, http.StatusOK)
return return
} }
if !hasAllPrivilege { if !hasAllPrivilege {
@ -110,7 +106,7 @@ func (h *APIHandler) HandleSearchActivityAction(w http.ResponseWriter, req *http
normalIndices []string normalIndices []string
) )
for _, index := range indices { for _, index := range indices {
if strings.Contains(index,"*") { if strings.Contains(index, "*") {
wildcardIndices = append(wildcardIndices, index) wildcardIndices = append(wildcardIndices, index)
continue continue
} }
@ -176,7 +172,7 @@ func (h *APIHandler) HandleSearchActivityAction(w http.ResponseWriter, req *http
var boolQuery = util.MapStr{ var boolQuery = util.MapStr{
"filter": filter, "filter": filter,
} }
if len(should) >0 { if len(should) > 0 {
boolQuery["should"] = should boolQuery["should"] = should
boolQuery["minimum_should_match"] = 1 boolQuery["minimum_should_match"] = 1
} }
@ -206,7 +202,7 @@ func (h *APIHandler) HandleSearchActivityAction(w http.ResponseWriter, req *http
response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(orm.GetWildcardIndexName(event.Activity{}), dsl) response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(orm.GetWildcardIndexName(event.Activity{}), dsl)
if err != nil { if err != nil {
resBody["error"] = err.Error() resBody["error"] = err.Error()
h.WriteJSON(w,resBody, http.StatusInternalServerError ) h.WriteJSON(w, resBody, http.StatusInternalServerError)
return return
} }
w.Write(response.RawResult.Body) w.Write(response.RawResult.Body)

8
modules/elastic/api/alias.go
View File

@ -33,9 +33,9 @@ import (
"net/http" "net/http"
) )
func (h *APIHandler) HandleAliasAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ func (h *APIHandler) HandleAliasAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
targetClusterID := ps.ByName("id") targetClusterID := ps.ByName("id")
exists,client,err:=h.GetClusterClient(targetClusterID) exists, client, err := h.GetClusterClient(targetClusterID)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
@ -43,8 +43,8 @@ func (h *APIHandler) HandleAliasAction(w http.ResponseWriter, req *http.Request,
return return
} }
if !exists{ if !exists {
errStr := fmt.Sprintf("cluster [%s] not found",targetClusterID) errStr := fmt.Sprintf("cluster [%s] not found", targetClusterID)
log.Error(errStr) log.Error(errStr)
h.WriteError(w, errStr, http.StatusInternalServerError) h.WriteError(w, errStr, http.StatusInternalServerError)
return return

50
modules/elastic/api/cluster_overview.go
View File

@ -145,7 +145,7 @@ func (h *APIHandler) FetchClusterInfo(w http.ResponseWriter, req *http.Request,
indexMetricItems := []GroupMetricItem{} indexMetricItems := []GroupMetricItem{}
metricItem := newMetricItem("cluster_indexing", 2, "cluster") metricItem := newMetricItem("cluster_indexing", 2, "cluster")
metricItem.OnlyPrimary = true metricItem.OnlyPrimary = true
indexMetricItems=append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "cluster_indexing", Key: "cluster_indexing",
Field: "payload.elasticsearch.node_stats.indices.indexing.index_total", Field: "payload.elasticsearch.node_stats.indices.indexing.index_total",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -156,7 +156,7 @@ func (h *APIHandler) FetchClusterInfo(w http.ResponseWriter, req *http.Request,
}) })
metricItem = newMetricItem("cluster_search", 2, "cluster") metricItem = newMetricItem("cluster_search", 2, "cluster")
indexMetricItems=append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "cluster_search", Key: "cluster_search",
Field: "payload.elasticsearch.node_stats.indices.search.query_total", Field: "payload.elasticsearch.node_stats.indices.search.query_total",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -166,7 +166,6 @@ func (h *APIHandler) FetchClusterInfo(w http.ResponseWriter, req *http.Request,
Units: "query/s", Units: "query/s",
}) })
clusterID := global.MustLookupString(elastic.GlobalSystemElasticsearchID) clusterID := global.MustLookupString(elastic.GlobalSystemElasticsearchID)
intervalField, err := getDateHistogramIntervalField(clusterID, bucketSizeStr) intervalField, err := getDateHistogramIntervalField(clusterID, bucketSizeStr)
if err != nil { if err != nil {
@ -200,23 +199,23 @@ func (h *APIHandler) FetchClusterInfo(w http.ResponseWriter, req *http.Request,
{ {
"range": util.MapStr{ "range": util.MapStr{
"timestamp": util.MapStr{ "timestamp": util.MapStr{
"gte": fmt.Sprintf("now-%ds", metricLen * bucketSize), "gte": fmt.Sprintf("now-%ds", metricLen*bucketSize),
}, },
}, },
}, },
}, },
}, },
} }
aggs:=map[string]interface{}{} aggs := map[string]interface{}{}
sumAggs := util.MapStr{} sumAggs := util.MapStr{}
for _,metricItem:=range indexMetricItems { for _, metricItem := range indexMetricItems {
leafAgg := util.MapStr{ leafAgg := util.MapStr{
"max":util.MapStr{ "max": util.MapStr{
"field": metricItem.Field, "field": metricItem.Field,
}, },
} }
var sumBucketPath = "term_node>"+ metricItem.ID var sumBucketPath = "term_node>" + metricItem.ID
aggs[metricItem.ID] = leafAgg aggs[metricItem.ID] = leafAgg
sumAggs[metricItem.ID] = util.MapStr{ sumAggs[metricItem.ID] = util.MapStr{
@ -224,22 +223,22 @@ func (h *APIHandler) FetchClusterInfo(w http.ResponseWriter, req *http.Request,
"buckets_path": sumBucketPath, "buckets_path": sumBucketPath,
}, },
} }
if metricItem.IsDerivative{ if metricItem.IsDerivative {
sumAggs[metricItem.ID+"_deriv"]=util.MapStr{ sumAggs[metricItem.ID+"_deriv"] = util.MapStr{
"derivative":util.MapStr{ "derivative": util.MapStr{
"buckets_path": metricItem.ID, "buckets_path": metricItem.ID,
}, },
} }
} }
} }
sumAggs["term_node"]= util.MapStr{ sumAggs["term_node"] = util.MapStr{
"terms": util.MapStr{ "terms": util.MapStr{
"field": "metadata.labels.node_id", "field": "metadata.labels.node_id",
"size": 1000, "size": 1000,
}, },
"aggs": aggs, "aggs": aggs,
} }
query["aggs"]= util.MapStr{ query["aggs"] = util.MapStr{
"group_by_level": util.MapStr{ "group_by_level": util.MapStr{
"terms": util.MapStr{ "terms": util.MapStr{
"field": "metadata.labels.cluster_uuid", "field": "metadata.labels.cluster_uuid",
@ -247,11 +246,11 @@ func (h *APIHandler) FetchClusterInfo(w http.ResponseWriter, req *http.Request,
}, },
"aggs": util.MapStr{ "aggs": util.MapStr{
"dates": util.MapStr{ "dates": util.MapStr{
"date_histogram":util.MapStr{ "date_histogram": util.MapStr{
"field": "timestamp", "field": "timestamp",
intervalField: bucketSizeStr, intervalField: bucketSizeStr,
}, },
"aggs":sumAggs, "aggs": sumAggs,
}, },
}, },
}, },
@ -279,12 +278,12 @@ func (h *APIHandler) FetchClusterInfo(w http.ResponseWriter, req *http.Request,
for _, line := range indexMetrics["cluster_indexing"].Lines { for _, line := range indexMetrics["cluster_indexing"].Lines {
// remove first metric dot // remove first metric dot
data := line.Data data := line.Data
if v, ok := data.([][]interface{}); ok && len(v)> 0 { if v, ok := data.([][]interface{}); ok && len(v) > 0 {
// remove first metric dot // remove first metric dot
temp := v[1:] temp := v[1:]
// // remove first last dot // // remove first last dot
if len(temp) > 0 { if len(temp) > 0 {
temp = temp[0: len(temp)-1] temp = temp[0 : len(temp)-1]
} }
data = temp data = temp
} }
@ -293,12 +292,12 @@ func (h *APIHandler) FetchClusterInfo(w http.ResponseWriter, req *http.Request,
searchMetricData := util.MapStr{} searchMetricData := util.MapStr{}
for _, line := range indexMetrics["cluster_search"].Lines { for _, line := range indexMetrics["cluster_search"].Lines {
data := line.Data data := line.Data
if v, ok := data.([][]interface{}); ok && len(v)> 0 { if v, ok := data.([][]interface{}); ok && len(v) > 0 {
// remove first metric dot // remove first metric dot
temp := v[1:] temp := v[1:]
// // remove first last dot // // remove first last dot
if len(temp) > 0 { if len(temp) > 0 {
temp = temp[0: len(temp)-1] temp = temp[0 : len(temp)-1]
} }
data = temp data = temp
} }
@ -633,7 +632,6 @@ func (h *APIHandler) GetClusterNodes(w http.ResponseWriter, req *http.Request, p
} }
} }
if v, ok := nodeID.(string); ok { if v, ok := nodeID.(string); ok {
nodeInfos[v] = util.MapStr{ nodeInfos[v] = util.MapStr{
"timestamp": hitM["timestamp"], "timestamp": hitM["timestamp"],
@ -902,7 +900,7 @@ func (h *APIHandler) getIndexQPS(clusterID string, bucketSizeInSeconds int) (map
}, },
"aggs": util.MapStr{ "aggs": util.MapStr{
"filter_pri": util.MapStr{ "filter_pri": util.MapStr{
"filter": util.MapStr{ "term": util.MapStr{ "payload.elasticsearch.shard_stats.routing.primary": true } }, "filter": util.MapStr{"term": util.MapStr{"payload.elasticsearch.shard_stats.routing.primary": true}},
"aggs": util.MapStr{ "aggs": util.MapStr{
"index_total": util.MapStr{ "index_total": util.MapStr{
"max": util.MapStr{ "max": util.MapStr{
@ -1405,11 +1403,11 @@ func (h *APIHandler) getClusterMonitorState(w http.ResponseWriter, req *http.Req
key := bk["key"].(string) key := bk["key"].(string)
if tv, ok := bk["max_timestamp"].(map[string]interface{}); ok { if tv, ok := bk["max_timestamp"].(map[string]interface{}); ok {
if collectionMode == elastic.ModeAgentless { if collectionMode == elastic.ModeAgentless {
if util.StringInArray([]string{ "index_stats", "cluster_health", "cluster_stats", "node_stats"}, key) { if util.StringInArray([]string{"index_stats", "cluster_health", "cluster_stats", "node_stats"}, key) {
ret[key] = getCollectionStats(tv["value"]) ret[key] = getCollectionStats(tv["value"])
} }
}else{ } else {
if util.StringInArray([]string{ "shard_stats", "cluster_health", "cluster_stats", "node_stats"}, key) { if util.StringInArray([]string{"shard_stats", "cluster_health", "cluster_stats", "node_stats"}, key) {
ret[key] = getCollectionStats(tv["value"]) ret[key] = getCollectionStats(tv["value"])
} }
} }
@ -1426,9 +1424,9 @@ func getCollectionStats(lastActiveAt interface{}) util.MapStr {
} }
if timestamp, ok := lastActiveAt.(float64); ok { if timestamp, ok := lastActiveAt.(float64); ok {
t := time.Unix(int64(timestamp/1000), 0) t := time.Unix(int64(timestamp/1000), 0)
if time.Now().Sub(t) > 5 * time.Minute { if time.Now().Sub(t) > 5*time.Minute {
stats["status"] = "warning" stats["status"] = "warning"
}else{ } else {
stats["status"] = "ok" stats["status"] = "ok"
} }
} }

35
modules/elastic/api/discover.go
View File

@ -39,7 +39,7 @@ import (
func (h *APIHandler) HandleEseSearchAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *APIHandler) HandleEseSearchAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
targetClusterID := ps.ByName("id") targetClusterID := ps.ByName("id")
exists,client,err:=h.GetClusterClient(targetClusterID) exists, client, err := h.GetClusterClient(targetClusterID)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
@ -47,14 +47,14 @@ func (h *APIHandler) HandleEseSearchAction(w http.ResponseWriter, req *http.Requ
return return
} }
if !exists{ if !exists {
errStr := fmt.Sprintf("cluster [%s] not found",targetClusterID) errStr := fmt.Sprintf("cluster [%s] not found", targetClusterID)
log.Error(errStr) log.Error(errStr)
h.WriteError(w, errStr, http.StatusNotFound) h.WriteError(w, errStr, http.StatusNotFound)
return return
} }
var reqParams = struct{ var reqParams = struct {
Index string `json:"index"` Index string `json:"index"`
Body map[string]interface{} `json:"body"` Body map[string]interface{} `json:"body"`
DistinctByField map[string]interface{} `json:"distinct_by_field"` DistinctByField map[string]interface{} `json:"distinct_by_field"`
@ -101,7 +101,7 @@ func (h *APIHandler) HandleEseSearchAction(w http.ResponseWriter, req *http.Requ
if qm, ok := query.(map[string]interface{}); ok { if qm, ok := query.(map[string]interface{}); ok {
filter, _ := util.MapStr(qm).GetValue("bool.filter") filter, _ := util.MapStr(qm).GetValue("bool.filter")
if fv, ok := filter.([]interface{}); ok{ if fv, ok := filter.([]interface{}); ok {
fv = append(fv, util.MapStr{ fv = append(fv, util.MapStr{
"script": util.MapStr{ "script": util.MapStr{
"script": util.MapStr{ "script": util.MapStr{
@ -184,7 +184,7 @@ func (h *APIHandler) HandleEseSearchAction(w http.ResponseWriter, req *http.Requ
} }
var cancel context.CancelFunc var cancel context.CancelFunc
// here add one second for network delay // here add one second for network delay
ctx, cancel = context.WithTimeout(context.Background(), du + time.Second) ctx, cancel = context.WithTimeout(context.Background(), du+time.Second)
defer cancel() defer cancel()
} }
@ -207,12 +207,10 @@ func (h *APIHandler) HandleEseSearchAction(w http.ResponseWriter, req *http.Requ
h.Write(w, searchRes.RawResult.Body) h.Write(w, searchRes.RawResult.Body)
} }
func (h *APIHandler) HandleValueSuggestionAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *APIHandler) HandleValueSuggestionAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody := map[string]interface{}{ resBody := map[string]interface{}{}
}
targetClusterID := ps.ByName("id") targetClusterID := ps.ByName("id")
exists,client,err:=h.GetClusterClient(targetClusterID) exists, client, err := h.GetClusterClient(targetClusterID)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
@ -221,13 +219,13 @@ func (h *APIHandler) HandleValueSuggestionAction(w http.ResponseWriter, req *htt
return return
} }
if !exists{ if !exists {
errStr := fmt.Sprintf("cluster [%s] not found",targetClusterID) errStr := fmt.Sprintf("cluster [%s] not found", targetClusterID)
h.WriteError(w, errStr, http.StatusNotFound) h.WriteError(w, errStr, http.StatusNotFound)
return return
} }
var reqParams = struct{ var reqParams = struct {
BoolFilter interface{} `json:"boolFilter"` BoolFilter interface{} `json:"boolFilter"`
FieldName string `json:"field"` FieldName string `json:"field"`
Query string `json:"query"` Query string `json:"query"`
@ -246,7 +244,7 @@ func (h *APIHandler) HandleValueSuggestionAction(w http.ResponseWriter, req *htt
indices, hasAll := h.GetAllowedIndices(req, targetClusterID) indices, hasAll := h.GetAllowedIndices(req, targetClusterID)
if !hasAll { if !hasAll {
if len(indices) == 0 { if len(indices) == 0 {
h.WriteJSON(w, values,http.StatusOK) h.WriteJSON(w, values, http.StatusOK)
return return
} }
boolQ["must"] = []util.MapStr{ boolQ["must"] = []util.MapStr{
@ -285,7 +283,7 @@ func (h *APIHandler) HandleValueSuggestionAction(w http.ResponseWriter, req *htt
for _, bucket := range searchRes.Aggregations["suggestions"].Buckets { for _, bucket := range searchRes.Aggregations["suggestions"].Buckets {
values = append(values, bucket["key"]) values = append(values, bucket["key"])
} }
h.WriteJSON(w, values,http.StatusOK) h.WriteJSON(w, values, http.StatusOK)
} }
func (h *APIHandler) HandleTraceIDSearchAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *APIHandler) HandleTraceIDSearchAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
@ -293,7 +291,7 @@ func (h *APIHandler) HandleTraceIDSearchAction(w http.ResponseWriter, req *http.
traceIndex := h.GetParameterOrDefault(req, "traceIndex", orm.GetIndexName(elastic.TraceMeta{})) traceIndex := h.GetParameterOrDefault(req, "traceIndex", orm.GetIndexName(elastic.TraceMeta{}))
traceField := h.GetParameterOrDefault(req, "traceField", "trace_id") traceField := h.GetParameterOrDefault(req, "traceField", "trace_id")
targetClusterID := ps.ByName("id") targetClusterID := ps.ByName("id")
exists,client,err:=h.GetClusterClient(targetClusterID) exists, client, err := h.GetClusterClient(targetClusterID)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
@ -301,8 +299,8 @@ func (h *APIHandler) HandleTraceIDSearchAction(w http.ResponseWriter, req *http.
return return
} }
if !exists{ if !exists {
errStr := fmt.Sprintf("cluster [%s] not found",targetClusterID) errStr := fmt.Sprintf("cluster [%s] not found", targetClusterID)
h.WriteError(w, errStr, http.StatusNotFound) h.WriteError(w, errStr, http.StatusNotFound)
return return
} }
@ -340,4 +338,3 @@ func (h *APIHandler) HandleTraceIDSearchAction(w http.ResponseWriter, req *http.
} }
h.WriteJSON(w, indexNames, http.StatusOK) h.WriteJSON(w, indexNames, http.StatusOK)
} }

12
modules/elastic/api/host.go
View File

@ -211,8 +211,7 @@ func (h *APIHandler) getDiscoverHosts(w http.ResponseWriter, req *http.Request,
func getHostSummary(agentIDs []string, metricName string, summary map[string]util.MapStr) error { func getHostSummary(agentIDs []string, metricName string, summary map[string]util.MapStr) error {
if summary == nil { if summary == nil {
summary = map[string]util.MapStr{ summary = map[string]util.MapStr{}
}
} }
if len(agentIDs) == 0 { if len(agentIDs) == 0 {
@ -506,8 +505,7 @@ func (h *APIHandler) FetchHostInfo(w http.ResponseWriter, req *http.Request, ps
for key, item := range hostMetrics { for key, item := range hostMetrics {
for _, line := range item.Lines { for _, line := range item.Lines {
if _, ok := networkMetrics[line.Metric.Label]; !ok { if _, ok := networkMetrics[line.Metric.Label]; !ok {
networkMetrics[line.Metric.Label] = util.MapStr{ networkMetrics[line.Metric.Label] = util.MapStr{}
}
} }
networkMetrics[line.Metric.Label][key] = line.Data networkMetrics[line.Metric.Label][key] = line.Data
} }
@ -687,15 +685,15 @@ const (
DiskUsedPercentMetricKey = "disk_used_percent" DiskUsedPercentMetricKey = "disk_used_percent"
SystemLoadMetricKey = "system_load" SystemLoadMetricKey = "system_load"
CPUIowaitMetricKey = "cpu_iowait" CPUIowaitMetricKey = "cpu_iowait"
SwapMemoryUsedPercentMetricKey= "swap_memory_used_percent" SwapMemoryUsedPercentMetricKey = "swap_memory_used_percent"
NetworkSummaryMetricKey = "network_summary" NetworkSummaryMetricKey = "network_summary"
NetworkPacketsSummaryMetricKey = "network_packets_summary" NetworkPacketsSummaryMetricKey = "network_packets_summary"
DiskReadRateMetricKey = "disk_read_rate" DiskReadRateMetricKey = "disk_read_rate"
DiskWriteRateMetricKey = "disk_write_rate" DiskWriteRateMetricKey = "disk_write_rate"
DiskPartitionUsageMetricKey = "disk_partition_usage" DiskPartitionUsageMetricKey = "disk_partition_usage"
NetworkInterfaceOutputRateMetricKey = "network_interface_output_rate" NetworkInterfaceOutputRateMetricKey = "network_interface_output_rate"
) )
func (h *APIHandler) GetSingleHostMetrics(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *APIHandler) GetSingleHostMetrics(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
hostID := ps.MustGetParameter("host_id") hostID := ps.MustGetParameter("host_id")
hostInfo := &host.HostInfo{} hostInfo := &host.HostInfo{}
@ -798,7 +796,7 @@ func (h *APIHandler) GetSingleHostMetrics(w http.ResponseWriter, req *http.Reque
metricItem.AddLine("Disk Write Rate", "Disk Write Rate", "network write rate of host.", "group1", "payload.host.diskio_summary.write.bytes", "max", bucketSizeStr, "%", "bytes", "0,0.[00]", "0,0.[00]", false, true) metricItem.AddLine("Disk Write Rate", "Disk Write Rate", "network write rate of host.", "group1", "payload.host.diskio_summary.write.bytes", "max", bucketSizeStr, "%", "bytes", "0,0.[00]", "0,0.[00]", false, true)
metricItems = append(metricItems, metricItem) metricItems = append(metricItems, metricItem)
case DiskPartitionUsageMetricKey, NetworkInterfaceOutputRateMetricKey: case DiskPartitionUsageMetricKey, NetworkInterfaceOutputRateMetricKey:
resBody["metrics"] , err = h.getGroupHostMetrics(ctx, hostInfo.AgentID, min, max, bucketSize, key) resBody["metrics"], err = h.getGroupHostMetrics(ctx, hostInfo.AgentID, min, max, bucketSize, key)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
h.WriteError(w, err, http.StatusInternalServerError) h.WriteError(w, err, http.StatusInternalServerError)

6
modules/elastic/api/ilm.go
View File

@ -35,7 +35,7 @@ import (
"net/http" "net/http"
) )
func (h *APIHandler) HandleGetILMPolicyAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ func (h *APIHandler) HandleGetILMPolicyAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
clusterID := ps.MustGetParameter("id") clusterID := ps.MustGetParameter("id")
esClient := elastic.GetClient(clusterID) esClient := elastic.GetClient(clusterID)
policies, err := esClient.GetILMPolicy("") policies, err := esClient.GetILMPolicy("")
@ -47,7 +47,7 @@ func (h *APIHandler) HandleGetILMPolicyAction(w http.ResponseWriter, req *http.R
h.WriteJSON(w, policies, http.StatusOK) h.WriteJSON(w, policies, http.StatusOK)
} }
func (h *APIHandler) HandleSaveILMPolicyAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ func (h *APIHandler) HandleSaveILMPolicyAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
clusterID := ps.MustGetParameter("id") clusterID := ps.MustGetParameter("id")
policy := ps.MustGetParameter("policy") policy := ps.MustGetParameter("policy")
esClient := elastic.GetClient(clusterID) esClient := elastic.GetClient(clusterID)
@ -66,7 +66,7 @@ func (h *APIHandler) HandleSaveILMPolicyAction(w http.ResponseWriter, req *http.
h.WriteAckOKJSON(w) h.WriteAckOKJSON(w)
} }
func (h *APIHandler) HandleDeleteILMPolicyAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ func (h *APIHandler) HandleDeleteILMPolicyAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
clusterID := ps.MustGetParameter("id") clusterID := ps.MustGetParameter("id")
policy := ps.MustGetParameter("policy") policy := ps.MustGetParameter("policy")
esClient := elastic.GetClient(clusterID) esClient := elastic.GetClient(clusterID)

216
modules/elastic/api/index_metrics.go
View File

@ -39,8 +39,8 @@ import (
"time" "time"
) )
//getClusterUUID reads the cluster uuid from metadata // getClusterUUID reads the cluster uuid from metadata
func (h *APIHandler) getClusterUUID(clusterID string) (string, error){ func (h *APIHandler) getClusterUUID(clusterID string) (string, error) {
meta := elastic.GetMetadata(clusterID) meta := elastic.GetMetadata(clusterID)
if meta == nil { if meta == nil {
return "", fmt.Errorf("metadata of cluster [%s] was not found", clusterID) return "", fmt.Errorf("metadata of cluster [%s] was not found", clusterID)
@ -48,16 +48,16 @@ func (h *APIHandler) getClusterUUID(clusterID string) (string, error){
return meta.Config.ClusterUUID, nil return meta.Config.ClusterUUID, nil
} }
func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clusterID string, bucketSize int, min, max int64, indexName string, top int, shardID string, metricKey string) (map[string]*common.MetricItem, error){ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clusterID string, bucketSize int, min, max int64, indexName string, top int, shardID string, metricKey string) (map[string]*common.MetricItem, error) {
bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
clusterUUID, err := h.getClusterUUID(clusterID) clusterUUID, err := h.getClusterUUID(clusterID)
if err != nil { if err != nil {
return nil, err return nil, err
} }
should := []util.MapStr{ should := []util.MapStr{
{ {
"term":util.MapStr{ "term": util.MapStr{
"metadata.labels.cluster_id":util.MapStr{ "metadata.labels.cluster_id": util.MapStr{
"value": clusterID, "value": clusterID,
}, },
}, },
@ -65,8 +65,8 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
} }
if clusterUUID != "" { if clusterUUID != "" {
should = append(should, util.MapStr{ should = append(should, util.MapStr{
"term":util.MapStr{ "term": util.MapStr{
"metadata.labels.cluster_uuid":util.MapStr{ "metadata.labels.cluster_uuid": util.MapStr{
"value": clusterUUID, "value": clusterUUID,
}, },
}, },
@ -107,11 +107,11 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
if !hasAllPrivilege && len(allowedIndices) == 0 { if !hasAllPrivilege && len(allowedIndices) == 0 {
return nil, nil return nil, nil
} }
if !hasAllPrivilege{ if !hasAllPrivilege {
namePattern := radix.Compile(allowedIndices...) namePattern := radix.Compile(allowedIndices...)
var filterNames []string var filterNames []string
for _, name := range indexNames { for _, name := range indexNames {
if namePattern.Match(name){ if namePattern.Match(name) {
filterNames = append(filterNames, name) filterNames = append(filterNames, name)
} }
} }
@ -122,7 +122,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
} }
top = len(indexNames) top = len(indexNames)
}else{ } else {
indexNames, err = h.getTopIndexName(req, clusterID, top, 15) indexNames, err = h.getTopIndexName(req, clusterID, top, 15)
if err != nil { if err != nil {
return nil, err return nil, err
@ -137,13 +137,13 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
} }
query:=map[string]interface{}{} query := map[string]interface{}{}
indexMetricItems := []GroupMetricItem{} indexMetricItems := []GroupMetricItem{}
switch metricKey { switch metricKey {
case v1.IndexStorageMetricKey: case v1.IndexStorageMetricKey:
//索引存储大小 //索引存储大小
indexStorageMetric := newMetricItem(v1.IndexStorageMetricKey, 1, StorageGroupKey) indexStorageMetric := newMetricItem(v1.IndexStorageMetricKey, 1, StorageGroupKey)
indexStorageMetric.AddAxi("Index storage","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) indexStorageMetric.AddAxi("Index storage", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "index_storage", Key: "index_storage",
Field: "payload.elasticsearch.shard_stats.store.size_in_bytes", Field: "payload.elasticsearch.shard_stats.store.size_in_bytes",
@ -155,9 +155,9 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case v1.SegmentCountMetricKey: case v1.SegmentCountMetricKey:
// segment 数量 // segment 数量
segmentCountMetric:=newMetricItem(v1.SegmentCountMetricKey, 15, StorageGroupKey) segmentCountMetric := newMetricItem(v1.SegmentCountMetricKey, 15, StorageGroupKey)
segmentCountMetric.AddAxi("segment count","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) segmentCountMetric.AddAxi("segment count", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
indexMetricItems=append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "segment_count", Key: "segment_count",
Field: "payload.elasticsearch.shard_stats.segments.count", Field: "payload.elasticsearch.shard_stats.segments.count",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -169,7 +169,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case v1.DocCountMetricKey: case v1.DocCountMetricKey:
//索引文档个数 //索引文档个数
docCountMetric := newMetricItem(v1.DocCountMetricKey, 2, DocumentGroupKey) docCountMetric := newMetricItem(v1.DocCountMetricKey, 2, DocumentGroupKey)
docCountMetric.AddAxi("Doc count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) docCountMetric.AddAxi("Doc count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "doc_count", Key: "doc_count",
@ -182,9 +182,9 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case v1.DocsDeletedMetricKey: case v1.DocsDeletedMetricKey:
// docs 删除数量 // docs 删除数量
docsDeletedMetric:=newMetricItem(v1.DocsDeletedMetricKey, 17, DocumentGroupKey) docsDeletedMetric := newMetricItem(v1.DocsDeletedMetricKey, 17, DocumentGroupKey)
docsDeletedMetric.AddAxi("docs deleted","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) docsDeletedMetric.AddAxi("docs deleted", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
indexMetricItems=append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "docs_deleted", Key: "docs_deleted",
Field: "payload.elasticsearch.shard_stats.docs.deleted", Field: "payload.elasticsearch.shard_stats.docs.deleted",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -196,7 +196,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case v1.QueryTimesMetricKey: case v1.QueryTimesMetricKey:
//查询次数 //查询次数
queryTimesMetric := newMetricItem("query_times", 2, OperationGroupKey) queryTimesMetric := newMetricItem("query_times", 2, OperationGroupKey)
queryTimesMetric.AddAxi("Query times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) queryTimesMetric.AddAxi("Query times", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "query_times", Key: "query_times",
@ -210,7 +210,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case v1.FetchTimesMetricKey: case v1.FetchTimesMetricKey:
//Fetch次数 //Fetch次数
fetchTimesMetric := newMetricItem(v1.FetchTimesMetricKey, 3, OperationGroupKey) fetchTimesMetric := newMetricItem(v1.FetchTimesMetricKey, 3, OperationGroupKey)
fetchTimesMetric.AddAxi("Fetch times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) fetchTimesMetric.AddAxi("Fetch times", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "fetch_times", Key: "fetch_times",
Field: "payload.elasticsearch.shard_stats.search.fetch_total", Field: "payload.elasticsearch.shard_stats.search.fetch_total",
@ -223,7 +223,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case v1.ScrollTimesMetricKey: case v1.ScrollTimesMetricKey:
//scroll 次数 //scroll 次数
scrollTimesMetric := newMetricItem(v1.ScrollTimesMetricKey, 4, OperationGroupKey) scrollTimesMetric := newMetricItem(v1.ScrollTimesMetricKey, 4, OperationGroupKey)
scrollTimesMetric.AddAxi("scroll times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) scrollTimesMetric.AddAxi("scroll times", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "scroll_times", Key: "scroll_times",
Field: "payload.elasticsearch.shard_stats.search.scroll_total", Field: "payload.elasticsearch.shard_stats.search.scroll_total",
@ -236,7 +236,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case v1.MergeTimesMetricKey: case v1.MergeTimesMetricKey:
//Merge次数 //Merge次数
mergeTimesMetric := newMetricItem(v1.MergeTimesMetricKey, 7, OperationGroupKey) mergeTimesMetric := newMetricItem(v1.MergeTimesMetricKey, 7, OperationGroupKey)
mergeTimesMetric.AddAxi("Merge times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) mergeTimesMetric.AddAxi("Merge times", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "merge_times", Key: "merge_times",
Field: "payload.elasticsearch.shard_stats.merges.total", Field: "payload.elasticsearch.shard_stats.merges.total",
@ -249,7 +249,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case v1.RefreshTimesMetricKey: case v1.RefreshTimesMetricKey:
//Refresh次数 //Refresh次数
refreshTimesMetric := newMetricItem(v1.RefreshTimesMetricKey, 5, OperationGroupKey) refreshTimesMetric := newMetricItem(v1.RefreshTimesMetricKey, 5, OperationGroupKey)
refreshTimesMetric.AddAxi("Refresh times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) refreshTimesMetric.AddAxi("Refresh times", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "refresh_times", Key: "refresh_times",
Field: "payload.elasticsearch.shard_stats.refresh.total", Field: "payload.elasticsearch.shard_stats.refresh.total",
@ -262,7 +262,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case v1.FlushTimesMetricKey: case v1.FlushTimesMetricKey:
//flush 次数 //flush 次数
flushTimesMetric := newMetricItem(v1.FlushTimesMetricKey, 6, OperationGroupKey) flushTimesMetric := newMetricItem(v1.FlushTimesMetricKey, 6, OperationGroupKey)
flushTimesMetric.AddAxi("flush times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) flushTimesMetric.AddAxi("flush times", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "flush_times", Key: "flush_times",
Field: "payload.elasticsearch.shard_stats.flush.total", Field: "payload.elasticsearch.shard_stats.flush.total",
@ -278,7 +278,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
if shardID == "" { if shardID == "" {
indexingRateMetric.OnlyPrimary = true indexingRateMetric.OnlyPrimary = true
} }
indexingRateMetric.AddAxi("Indexing rate","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) indexingRateMetric.AddAxi("Indexing rate", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "indexing_rate", Key: "indexing_rate",
Field: "payload.elasticsearch.shard_stats.indexing.index_total", Field: "payload.elasticsearch.shard_stats.indexing.index_total",
@ -293,7 +293,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
if shardID == "" { if shardID == "" {
indexingBytesMetric.OnlyPrimary = true indexingBytesMetric.OnlyPrimary = true
} }
indexingBytesMetric.AddAxi("Indexing bytes","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) indexingBytesMetric.AddAxi("Indexing bytes", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "indexing_bytes", Key: "indexing_bytes",
Field: "payload.elasticsearch.shard_stats.store.size_in_bytes", Field: "payload.elasticsearch.shard_stats.store.size_in_bytes",
@ -309,13 +309,13 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
if shardID == "" { if shardID == "" {
indexingLatencyMetric.OnlyPrimary = true indexingLatencyMetric.OnlyPrimary = true
} }
indexingLatencyMetric.AddAxi("Indexing latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) indexingLatencyMetric.AddAxi("Indexing latency", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "indexing_latency", Key: "indexing_latency",
Field: "payload.elasticsearch.shard_stats.indexing.index_time_in_millis", Field: "payload.elasticsearch.shard_stats.indexing.index_time_in_millis",
Field2: "payload.elasticsearch.shard_stats.indexing.index_total", Field2: "payload.elasticsearch.shard_stats.indexing.index_total",
Calc: func(value, value2 float64) float64 { Calc: func(value, value2 float64) float64 {
return value/value2 return value / value2
}, },
ID: util.GetUUID(), ID: util.GetUUID(),
IsDerivative: true, IsDerivative: true,
@ -326,13 +326,13 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case v1.QueryLatencyMetricKey: case v1.QueryLatencyMetricKey:
//查询时延 //查询时延
queryLatencyMetric := newMetricItem(v1.QueryLatencyMetricKey, 2, LatencyGroupKey) queryLatencyMetric := newMetricItem(v1.QueryLatencyMetricKey, 2, LatencyGroupKey)
queryLatencyMetric.AddAxi("Query latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) queryLatencyMetric.AddAxi("Query latency", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "query_latency", Key: "query_latency",
Field: "payload.elasticsearch.shard_stats.search.query_time_in_millis", Field: "payload.elasticsearch.shard_stats.search.query_time_in_millis",
Field2: "payload.elasticsearch.shard_stats.search.query_total", Field2: "payload.elasticsearch.shard_stats.search.query_total",
Calc: func(value, value2 float64) float64 { Calc: func(value, value2 float64) float64 {
return value/value2 return value / value2
}, },
ID: util.GetUUID(), ID: util.GetUUID(),
IsDerivative: true, IsDerivative: true,
@ -343,13 +343,13 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case FetchLatencyMetricKey: case FetchLatencyMetricKey:
//fetch时延 //fetch时延
fetchLatencyMetric := newMetricItem(v1.FetchLatencyMetricKey, 3, LatencyGroupKey) fetchLatencyMetric := newMetricItem(v1.FetchLatencyMetricKey, 3, LatencyGroupKey)
fetchLatencyMetric.AddAxi("Fetch latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) fetchLatencyMetric.AddAxi("Fetch latency", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "fetch_latency", Key: "fetch_latency",
Field: "payload.elasticsearch.shard_stats.search.fetch_time_in_millis", Field: "payload.elasticsearch.shard_stats.search.fetch_time_in_millis",
Field2: "payload.elasticsearch.shard_stats.search.fetch_total", Field2: "payload.elasticsearch.shard_stats.search.fetch_total",
Calc: func(value, value2 float64) float64 { Calc: func(value, value2 float64) float64 {
return value/value2 return value / value2
}, },
ID: util.GetUUID(), ID: util.GetUUID(),
IsDerivative: true, IsDerivative: true,
@ -360,13 +360,13 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case v1.MergeLatencyMetricKey: case v1.MergeLatencyMetricKey:
//merge时延 //merge时延
mergeLatencyMetric := newMetricItem(v1.MergeLatencyMetricKey, 7, LatencyGroupKey) mergeLatencyMetric := newMetricItem(v1.MergeLatencyMetricKey, 7, LatencyGroupKey)
mergeLatencyMetric.AddAxi("Merge latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) mergeLatencyMetric.AddAxi("Merge latency", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "merge_latency", Key: "merge_latency",
Field: "payload.elasticsearch.shard_stats.merges.total_time_in_millis", Field: "payload.elasticsearch.shard_stats.merges.total_time_in_millis",
Field2: "payload.elasticsearch.shard_stats.merges.total", Field2: "payload.elasticsearch.shard_stats.merges.total",
Calc: func(value, value2 float64) float64 { Calc: func(value, value2 float64) float64 {
return value/value2 return value / value2
}, },
ID: util.GetUUID(), ID: util.GetUUID(),
IsDerivative: true, IsDerivative: true,
@ -377,13 +377,13 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case RefreshLatencyMetricKey: case RefreshLatencyMetricKey:
//refresh时延 //refresh时延
refreshLatencyMetric := newMetricItem(v1.RefreshLatencyMetricKey, 5, LatencyGroupKey) refreshLatencyMetric := newMetricItem(v1.RefreshLatencyMetricKey, 5, LatencyGroupKey)
refreshLatencyMetric.AddAxi("Refresh latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) refreshLatencyMetric.AddAxi("Refresh latency", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "refresh_latency", Key: "refresh_latency",
Field: "payload.elasticsearch.shard_stats.refresh.total_time_in_millis", Field: "payload.elasticsearch.shard_stats.refresh.total_time_in_millis",
Field2: "payload.elasticsearch.shard_stats.refresh.total", Field2: "payload.elasticsearch.shard_stats.refresh.total",
Calc: func(value, value2 float64) float64 { Calc: func(value, value2 float64) float64 {
return value/value2 return value / value2
}, },
ID: util.GetUUID(), ID: util.GetUUID(),
IsDerivative: true, IsDerivative: true,
@ -394,13 +394,13 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case v1.ScrollLatencyMetricKey: case v1.ScrollLatencyMetricKey:
//scroll时延 //scroll时延
scrollLatencyMetric := newMetricItem(v1.ScrollLatencyMetricKey, 4, LatencyGroupKey) scrollLatencyMetric := newMetricItem(v1.ScrollLatencyMetricKey, 4, LatencyGroupKey)
scrollLatencyMetric.AddAxi("Scroll Latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) scrollLatencyMetric.AddAxi("Scroll Latency", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "scroll_latency", Key: "scroll_latency",
Field: "payload.elasticsearch.shard_stats.search.scroll_time_in_millis", Field: "payload.elasticsearch.shard_stats.search.scroll_time_in_millis",
Field2: "payload.elasticsearch.shard_stats.search.scroll_total", Field2: "payload.elasticsearch.shard_stats.search.scroll_total",
Calc: func(value, value2 float64) float64 { Calc: func(value, value2 float64) float64 {
return value/value2 return value / value2
}, },
ID: util.GetUUID(), ID: util.GetUUID(),
IsDerivative: true, IsDerivative: true,
@ -411,13 +411,13 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case v1.FlushLatencyMetricKey: case v1.FlushLatencyMetricKey:
//flush 时延 //flush 时延
flushLatencyMetric := newMetricItem(v1.FlushLatencyMetricKey, 6, LatencyGroupKey) flushLatencyMetric := newMetricItem(v1.FlushLatencyMetricKey, 6, LatencyGroupKey)
flushLatencyMetric.AddAxi("Flush latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) flushLatencyMetric.AddAxi("Flush latency", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "flush_latency", Key: "flush_latency",
Field: "payload.elasticsearch.shard_stats.flush.total_time_in_millis", Field: "payload.elasticsearch.shard_stats.flush.total_time_in_millis",
Field2: "payload.elasticsearch.shard_stats.flush.total", Field2: "payload.elasticsearch.shard_stats.flush.total",
Calc: func(value, value2 float64) float64 { Calc: func(value, value2 float64) float64 {
return value/value2 return value / value2
}, },
ID: util.GetUUID(), ID: util.GetUUID(),
IsDerivative: true, IsDerivative: true,
@ -428,7 +428,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case v1.QueryCacheMetricKey: case v1.QueryCacheMetricKey:
//queryCache //queryCache
queryCacheMetric := newMetricItem(v1.QueryCacheMetricKey, 1, CacheGroupKey) queryCacheMetric := newMetricItem(v1.QueryCacheMetricKey, 1, CacheGroupKey)
queryCacheMetric.AddAxi("Query cache","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) queryCacheMetric.AddAxi("Query cache", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "query_cache", Key: "query_cache",
Field: "payload.elasticsearch.shard_stats.query_cache.memory_size_in_bytes", Field: "payload.elasticsearch.shard_stats.query_cache.memory_size_in_bytes",
@ -441,7 +441,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case v1.RequestCacheMetricKey: case v1.RequestCacheMetricKey:
//requestCache //requestCache
requestCacheMetric := newMetricItem(v1.RequestCacheMetricKey, 2, CacheGroupKey) requestCacheMetric := newMetricItem(v1.RequestCacheMetricKey, 2, CacheGroupKey)
requestCacheMetric.AddAxi("request cache","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) requestCacheMetric.AddAxi("request cache", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "request_cache", Key: "request_cache",
Field: "payload.elasticsearch.shard_stats.request_cache.memory_size_in_bytes", Field: "payload.elasticsearch.shard_stats.request_cache.memory_size_in_bytes",
@ -453,9 +453,9 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case v1.RequestCacheHitMetricKey: case v1.RequestCacheHitMetricKey:
// Request Cache Hit // Request Cache Hit
requestCacheHitMetric:=newMetricItem(v1.RequestCacheHitMetricKey, 6, CacheGroupKey) requestCacheHitMetric := newMetricItem(v1.RequestCacheHitMetricKey, 6, CacheGroupKey)
requestCacheHitMetric.AddAxi("request cache hit","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) requestCacheHitMetric.AddAxi("request cache hit", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
indexMetricItems=append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "request_cache_hit", Key: "request_cache_hit",
Field: "payload.elasticsearch.shard_stats.request_cache.hit_count", Field: "payload.elasticsearch.shard_stats.request_cache.hit_count",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -466,9 +466,9 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case v1.RequestCacheMissMetricKey: case v1.RequestCacheMissMetricKey:
// Request Cache Miss // Request Cache Miss
requestCacheMissMetric:=newMetricItem(v1.RequestCacheMissMetricKey, 8, CacheGroupKey) requestCacheMissMetric := newMetricItem(v1.RequestCacheMissMetricKey, 8, CacheGroupKey)
requestCacheMissMetric.AddAxi("request cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) requestCacheMissMetric.AddAxi("request cache miss", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
indexMetricItems=append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "request_cache_miss", Key: "request_cache_miss",
Field: "payload.elasticsearch.shard_stats.request_cache.miss_count", Field: "payload.elasticsearch.shard_stats.request_cache.miss_count",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -479,9 +479,9 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case v1.QueryCacheCountMetricKey: case v1.QueryCacheCountMetricKey:
// Query Cache Count // Query Cache Count
queryCacheCountMetric:=newMetricItem(v1.QueryCacheCountMetricKey, 4, CacheGroupKey) queryCacheCountMetric := newMetricItem(v1.QueryCacheCountMetricKey, 4, CacheGroupKey)
queryCacheCountMetric.AddAxi("query cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) queryCacheCountMetric.AddAxi("query cache miss", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
indexMetricItems=append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "query_cache_count", Key: "query_cache_count",
Field: "payload.elasticsearch.shard_stats.query_cache.cache_count", Field: "payload.elasticsearch.shard_stats.query_cache.cache_count",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -492,9 +492,9 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case v1.QueryCacheHitMetricKey: case v1.QueryCacheHitMetricKey:
// Query Cache Miss // Query Cache Miss
queryCacheHitMetric:=newMetricItem(v1.QueryCacheHitMetricKey, 5, CacheGroupKey) queryCacheHitMetric := newMetricItem(v1.QueryCacheHitMetricKey, 5, CacheGroupKey)
queryCacheHitMetric.AddAxi("query cache hit","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) queryCacheHitMetric.AddAxi("query cache hit", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
indexMetricItems=append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "query_cache_hit", Key: "query_cache_hit",
Field: "payload.elasticsearch.shard_stats.query_cache.hit_count", Field: "payload.elasticsearch.shard_stats.query_cache.hit_count",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -505,9 +505,9 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case v1.QueryCacheMissMetricKey: case v1.QueryCacheMissMetricKey:
// Query Cache Miss // Query Cache Miss
queryCacheMissMetric:=newMetricItem(v1.QueryCacheMissMetricKey, 7, CacheGroupKey) queryCacheMissMetric := newMetricItem(v1.QueryCacheMissMetricKey, 7, CacheGroupKey)
queryCacheMissMetric.AddAxi("query cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) queryCacheMissMetric.AddAxi("query cache miss", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
indexMetricItems=append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "query_cache_miss", Key: "query_cache_miss",
Field: "payload.elasticsearch.shard_stats.query_cache.miss_count", Field: "payload.elasticsearch.shard_stats.query_cache.miss_count",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -518,9 +518,9 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case v1.FielddataCacheMetricKey: case v1.FielddataCacheMetricKey:
// Fielddata内存占用大小 // Fielddata内存占用大小
fieldDataCacheMetric:=newMetricItem(v1.FielddataCacheMetricKey, 3, CacheGroupKey) fieldDataCacheMetric := newMetricItem(v1.FielddataCacheMetricKey, 3, CacheGroupKey)
fieldDataCacheMetric.AddAxi("FieldData Cache","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) fieldDataCacheMetric.AddAxi("FieldData Cache", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, true)
indexMetricItems=append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "fielddata_cache", Key: "fielddata_cache",
Field: "payload.elasticsearch.shard_stats.fielddata.memory_size_in_bytes", Field: "payload.elasticsearch.shard_stats.fielddata.memory_size_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -532,7 +532,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case v1.SegmentMemoryMetricKey: case v1.SegmentMemoryMetricKey:
//segment memory //segment memory
segmentMemoryMetric := newMetricItem(v1.SegmentMemoryMetricKey, 13, MemoryGroupKey) segmentMemoryMetric := newMetricItem(v1.SegmentMemoryMetricKey, 13, MemoryGroupKey)
segmentMemoryMetric.AddAxi("Segment memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) segmentMemoryMetric.AddAxi("Segment memory", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "segment_memory", Key: "segment_memory",
Field: "payload.elasticsearch.shard_stats.segments.memory_in_bytes", Field: "payload.elasticsearch.shard_stats.segments.memory_in_bytes",
@ -545,7 +545,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case v1.SegmentDocValuesMemoryMetricKey: case v1.SegmentDocValuesMemoryMetricKey:
//segment doc values memory //segment doc values memory
docValuesMemoryMetric := newMetricItem(v1.SegmentDocValuesMemoryMetricKey, 13, MemoryGroupKey) docValuesMemoryMetric := newMetricItem(v1.SegmentDocValuesMemoryMetricKey, 13, MemoryGroupKey)
docValuesMemoryMetric.AddAxi("Segment Doc values Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) docValuesMemoryMetric.AddAxi("Segment Doc values Memory", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "segment_doc_values_memory", Key: "segment_doc_values_memory",
Field: "payload.elasticsearch.shard_stats.segments.doc_values_memory_in_bytes", Field: "payload.elasticsearch.shard_stats.segments.doc_values_memory_in_bytes",
@ -558,7 +558,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case v1.SegmentTermsMemoryMetricKey: case v1.SegmentTermsMemoryMetricKey:
//segment terms memory //segment terms memory
termsMemoryMetric := newMetricItem(v1.SegmentTermsMemoryMetricKey, 13, MemoryGroupKey) termsMemoryMetric := newMetricItem(v1.SegmentTermsMemoryMetricKey, 13, MemoryGroupKey)
termsMemoryMetric.AddAxi("Segment Terms Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) termsMemoryMetric.AddAxi("Segment Terms Memory", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "segment_terms_memory", Key: "segment_terms_memory",
Field: "payload.elasticsearch.shard_stats.segments.terms_memory_in_bytes", Field: "payload.elasticsearch.shard_stats.segments.terms_memory_in_bytes",
@ -571,7 +571,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case v1.SegmentFieldsMemoryMetricKey: case v1.SegmentFieldsMemoryMetricKey:
//segment fields memory //segment fields memory
fieldsMemoryMetric := newMetricItem(v1.SegmentFieldsMemoryMetricKey, 13, MemoryGroupKey) fieldsMemoryMetric := newMetricItem(v1.SegmentFieldsMemoryMetricKey, 13, MemoryGroupKey)
fieldsMemoryMetric.AddAxi("Segment Fields Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) fieldsMemoryMetric.AddAxi("Segment Fields Memory", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "segment_fields_memory", Key: "segment_fields_memory",
Field: "payload.elasticsearch.index_stats.total.segments.stored_fields_memory_in_bytes", Field: "payload.elasticsearch.index_stats.total.segments.stored_fields_memory_in_bytes",
@ -583,9 +583,9 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case v1.SegmentIndexWriterMemoryMetricKey: case v1.SegmentIndexWriterMemoryMetricKey:
// segment index writer memory // segment index writer memory
segmentIndexWriterMemoryMetric:=newMetricItem(v1.SegmentIndexWriterMemoryMetricKey, 16, MemoryGroupKey) segmentIndexWriterMemoryMetric := newMetricItem(v1.SegmentIndexWriterMemoryMetricKey, 16, MemoryGroupKey)
segmentIndexWriterMemoryMetric.AddAxi("segment doc values memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) segmentIndexWriterMemoryMetric.AddAxi("segment doc values memory", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, true)
indexMetricItems=append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "segment_index_writer_memory", Key: "segment_index_writer_memory",
Field: "payload.elasticsearch.shard_stats.segments.index_writer_memory_in_bytes", Field: "payload.elasticsearch.shard_stats.segments.index_writer_memory_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -596,9 +596,9 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case v1.SegmentTermVectorsMemoryMetricKey: case v1.SegmentTermVectorsMemoryMetricKey:
// segment term vectors memory // segment term vectors memory
segmentTermVectorsMemoryMetric:=newMetricItem(v1.SegmentTermVectorsMemoryMetricKey, 16, MemoryGroupKey) segmentTermVectorsMemoryMetric := newMetricItem(v1.SegmentTermVectorsMemoryMetricKey, 16, MemoryGroupKey)
segmentTermVectorsMemoryMetric.AddAxi("segment term vectors memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) segmentTermVectorsMemoryMetric.AddAxi("segment term vectors memory", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, true)
indexMetricItems=append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "segment_term_vectors_memory", Key: "segment_term_vectors_memory",
Field: "payload.elasticsearch.shard_stats.segments.term_vectors_memory_in_bytes", Field: "payload.elasticsearch.shard_stats.segments.term_vectors_memory_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -609,7 +609,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case v1.SegmentNormsMetricKey: case v1.SegmentNormsMetricKey:
segmentNormsMetric := newMetricItem(v1.SegmentNormsMetricKey, 17, MemoryGroupKey) segmentNormsMetric := newMetricItem(v1.SegmentNormsMetricKey, 17, MemoryGroupKey)
segmentNormsMetric.AddAxi("Segment norms memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) segmentNormsMetric.AddAxi("Segment norms memory", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: v1.SegmentNormsMetricKey, Key: v1.SegmentNormsMetricKey,
Field: "payload.elasticsearch.shard_stats.segments.norms_memory_in_bytes", Field: "payload.elasticsearch.shard_stats.segments.norms_memory_in_bytes",
@ -621,7 +621,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case v1.SegmentPointsMetricKey: case v1.SegmentPointsMetricKey:
segmentPointsMetric := newMetricItem(v1.SegmentPointsMetricKey, 18, MemoryGroupKey) segmentPointsMetric := newMetricItem(v1.SegmentPointsMetricKey, 18, MemoryGroupKey)
segmentPointsMetric.AddAxi("Segment points memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) segmentPointsMetric.AddAxi("Segment points memory", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: v1.SegmentPointsMetricKey, Key: v1.SegmentPointsMetricKey,
Field: "payload.elasticsearch.shard_stats.segments.points_memory_in_bytes", Field: "payload.elasticsearch.shard_stats.segments.points_memory_in_bytes",
@ -633,7 +633,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case v1.VersionMapMetricKey: case v1.VersionMapMetricKey:
segmentVersionMapMetric := newMetricItem(v1.VersionMapMetricKey, 18, MemoryGroupKey) segmentVersionMapMetric := newMetricItem(v1.VersionMapMetricKey, 18, MemoryGroupKey)
segmentVersionMapMetric.AddAxi("Segment version map memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) segmentVersionMapMetric.AddAxi("Segment version map memory", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: v1.VersionMapMetricKey, Key: v1.VersionMapMetricKey,
Field: "payload.elasticsearch.shard_stats.segments.version_map_memory_in_bytes", Field: "payload.elasticsearch.shard_stats.segments.version_map_memory_in_bytes",
@ -645,7 +645,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case v1.FixedBitSetMetricKey: case v1.FixedBitSetMetricKey:
segmentFixedBitSetMetric := newMetricItem(v1.FixedBitSetMetricKey, 18, MemoryGroupKey) segmentFixedBitSetMetric := newMetricItem(v1.FixedBitSetMetricKey, 18, MemoryGroupKey)
segmentFixedBitSetMetric.AddAxi("Segment fixed bit set memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) segmentFixedBitSetMetric.AddAxi("Segment fixed bit set memory", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: v1.FixedBitSetMetricKey, Key: v1.FixedBitSetMetricKey,
Field: "payload.elasticsearch.shard_stats.segments.fixed_bit_set_memory_in_bytes", Field: "payload.elasticsearch.shard_stats.segments.fixed_bit_set_memory_in_bytes",
@ -657,18 +657,17 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
} }
aggs := map[string]interface{}{}
aggs:=map[string]interface{}{}
sumAggs := util.MapStr{} sumAggs := util.MapStr{}
for _,metricItem:=range indexMetricItems { for _, metricItem := range indexMetricItems {
leafAgg := util.MapStr{ leafAgg := util.MapStr{
"max":util.MapStr{ "max": util.MapStr{
"field": metricItem.Field, "field": metricItem.Field,
}, },
} }
var sumBucketPath = "term_shard>"+ metricItem.ID var sumBucketPath = "term_shard>" + metricItem.ID
aggs[metricItem.ID]= leafAgg aggs[metricItem.ID] = leafAgg
sumAggs[metricItem.ID] = util.MapStr{ sumAggs[metricItem.ID] = util.MapStr{
"sum_bucket": util.MapStr{ "sum_bucket": util.MapStr{
@ -676,36 +675,36 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}, },
} }
if metricItem.Field2 != ""{ if metricItem.Field2 != "" {
leafAgg2 := util.MapStr{ leafAgg2 := util.MapStr{
"max":util.MapStr{ "max": util.MapStr{
"field": metricItem.Field2, "field": metricItem.Field2,
}, },
} }
aggs[metricItem.ID+"_field2"] = leafAgg2 aggs[metricItem.ID+"_field2"] = leafAgg2
sumAggs[metricItem.ID + "_field2"] = util.MapStr{ sumAggs[metricItem.ID+"_field2"] = util.MapStr{
"sum_bucket": util.MapStr{ "sum_bucket": util.MapStr{
"buckets_path": sumBucketPath + "_field2", "buckets_path": sumBucketPath + "_field2",
}, },
} }
} }
if metricItem.IsDerivative{ if metricItem.IsDerivative {
sumAggs[metricItem.ID+"_deriv"]=util.MapStr{ sumAggs[metricItem.ID+"_deriv"] = util.MapStr{
"derivative":util.MapStr{ "derivative": util.MapStr{
"buckets_path": metricItem.ID, "buckets_path": metricItem.ID,
}, },
} }
if metricItem.Field2 != "" { if metricItem.Field2 != "" {
sumAggs[metricItem.ID + "_deriv_field2"]=util.MapStr{ sumAggs[metricItem.ID+"_deriv_field2"] = util.MapStr{
"derivative":util.MapStr{ "derivative": util.MapStr{
"buckets_path": metricItem.ID + "_field2", "buckets_path": metricItem.ID + "_field2",
}, },
} }
} }
} }
} }
sumAggs["term_shard"]= util.MapStr{ sumAggs["term_shard"] = util.MapStr{
"terms": util.MapStr{ "terms": util.MapStr{
"field": "metadata.labels.shard_id", "field": "metadata.labels.shard_id",
"size": 10000, "size": 10000,
@ -728,7 +727,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}, },
}) })
} }
query["query"]=util.MapStr{ query["query"] = util.MapStr{
"bool": util.MapStr{ "bool": util.MapStr{
"must": must, "must": must,
"minimum_should_match": 1, "minimum_should_match": 1,
@ -754,8 +753,8 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}, },
}, },
} }
query["size"]=0 query["size"] = 0
query["aggs"]= util.MapStr{ query["aggs"] = util.MapStr{
"group_by_level": util.MapStr{ "group_by_level": util.MapStr{
"terms": util.MapStr{ "terms": util.MapStr{
"field": "metadata.labels.index_name", "field": "metadata.labels.index_name",
@ -767,11 +766,11 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}, },
"aggs": util.MapStr{ "aggs": util.MapStr{
"dates": util.MapStr{ "dates": util.MapStr{
"date_histogram":util.MapStr{ "date_histogram": util.MapStr{
"field": "timestamp", "field": "timestamp",
intervalField: bucketSizeStr, intervalField: bucketSizeStr,
}, },
"aggs":sumAggs, "aggs": sumAggs,
}, },
"max_store_bucket_sort": util.MapStr{ "max_store_bucket_sort": util.MapStr{
"bucket_sort": util.MapStr{ "bucket_sort": util.MapStr{
@ -805,7 +804,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
} }
func (h *APIHandler) getTopIndexName(req *http.Request, clusterID string, top int, lastMinutes int) ([]string, error){ func (h *APIHandler) getTopIndexName(req *http.Request, clusterID string, top int, lastMinutes int) ([]string, error) {
ver := h.Client().GetVersion() ver := h.Client().GetVersion()
cr, _ := util.VersionCompare(ver.Number, "6.1") cr, _ := util.VersionCompare(ver.Number, "6.1")
if (ver.Distribution == "" || ver.Distribution == elastic.Elasticsearch) && cr == -1 { if (ver.Distribution == "" || ver.Distribution == elastic.Elasticsearch) && cr == -1 {
@ -813,8 +812,8 @@ func (h *APIHandler) getTopIndexName(req *http.Request, clusterID string, top in
} }
var ( var (
now = time.Now() now = time.Now()
max = now.UnixNano()/1e6 max = now.UnixNano() / 1e6
min = now.Add(-time.Duration(lastMinutes) * time.Minute).UnixNano()/1e6 min = now.Add(-time.Duration(lastMinutes)*time.Minute).UnixNano() / 1e6
) )
clusterUUID, err := h.getClusterUUID(clusterID) clusterUUID, err := h.getClusterUUID(clusterID)
if err != nil { if err != nil {
@ -1005,20 +1004,20 @@ func (h *APIHandler) getTopIndexName(req *http.Request, clusterID string, top in
}, },
}, },
} }
response,err:=elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(),util.MustToJSONBytes(query)) response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query))
if err!=nil{ if err != nil {
log.Error(err) log.Error(err)
return nil, err return nil, err
} }
var maxQpsKVS = map[string] float64{} var maxQpsKVS = map[string]float64{}
for _, agg := range response.Aggregations { for _, agg := range response.Aggregations {
for _, bk := range agg.Buckets { for _, bk := range agg.Buckets {
key := bk["key"].(string) key := bk["key"].(string)
if maxQps, ok := bk["max_qps"].(map[string]interface{}); ok { if maxQps, ok := bk["max_qps"].(map[string]interface{}); ok {
val := maxQps["value"].(float64) val := maxQps["value"].(float64)
if _, ok = maxQpsKVS[key] ; ok { if _, ok = maxQpsKVS[key]; ok {
maxQpsKVS[key] = maxQpsKVS[key] + val maxQpsKVS[key] = maxQpsKVS[key] + val
}else{ } else {
maxQpsKVS[key] = val maxQpsKVS[key] = val
} }
} }
@ -1039,7 +1038,7 @@ func (h *APIHandler) getTopIndexName(req *http.Request, clusterID string, top in
length = len(qpsValues) length = len(qpsValues)
} }
indexNames := []string{} indexNames := []string{}
for i := 0; i <length; i++ { for i := 0; i < length; i++ {
indexNames = append(indexNames, qpsValues[i].Key) indexNames = append(indexNames, qpsValues[i].Key)
} }
return indexNames, nil return indexNames, nil
@ -1050,12 +1049,13 @@ type TopTerm struct {
Value float64 Value float64
} }
type TopTermOrder []TopTerm type TopTermOrder []TopTerm
func (t TopTermOrder) Len() int{
func (t TopTermOrder) Len() int {
return len(t) return len(t)
} }
func (t TopTermOrder) Less(i, j int) bool{ func (t TopTermOrder) Less(i, j int) bool {
return t[i].Value > t[j].Value //desc return t[i].Value > t[j].Value //desc
} }
func (t TopTermOrder) Swap(i, j int){ func (t TopTermOrder) Swap(i, j int) {
t[i], t[j] = t[j], t[i] t[i], t[j] = t[j], t[i]
} }

143
modules/elastic/api/index_overview.go
View File

@ -46,8 +46,8 @@ import (
) )
func (h *APIHandler) SearchIndexMetadata(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *APIHandler) SearchIndexMetadata(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody:=util.MapStr{} resBody := util.MapStr{}
reqBody := struct{ reqBody := struct {
Keyword string `json:"keyword"` Keyword string `json:"keyword"`
Size int `json:"size"` Size int `json:"size"`
From int `json:"from"` From int `json:"from"`
@ -60,7 +60,7 @@ func (h *APIHandler) SearchIndexMetadata(w http.ResponseWriter, req *http.Reques
err := h.DecodeJSON(req, &reqBody) err := h.DecodeJSON(req, &reqBody)
if err != nil { if err != nil {
resBody["error"] = err.Error() resBody["error"] = err.Error()
h.WriteJSON(w,resBody, http.StatusInternalServerError ) h.WriteJSON(w, resBody, http.StatusInternalServerError)
return return
} }
aggs := elastic.BuildSearchTermAggregations(reqBody.Aggregations) aggs := elastic.BuildSearchTermAggregations(reqBody.Aggregations)
@ -80,7 +80,7 @@ func (h *APIHandler) SearchIndexMetadata(w http.ResponseWriter, req *http.Reques
} }
filter := elastic.BuildSearchTermFilter(reqBody.Filter) filter := elastic.BuildSearchTermFilter(reqBody.Filter)
var should []util.MapStr var should []util.MapStr
if reqBody.SearchField != ""{ if reqBody.SearchField != "" {
should = []util.MapStr{ should = []util.MapStr{
{ {
"prefix": util.MapStr{ "prefix": util.MapStr{
@ -103,8 +103,8 @@ func (h *APIHandler) SearchIndexMetadata(w http.ResponseWriter, req *http.Reques
}, },
}, },
} }
}else{ } else {
if reqBody.Keyword != ""{ if reqBody.Keyword != "" {
should = []util.MapStr{ should = []util.MapStr{
{ {
"prefix": util.MapStr{ "prefix": util.MapStr{
@ -149,15 +149,13 @@ func (h *APIHandler) SearchIndexMetadata(w http.ResponseWriter, req *http.Reques
} }
} }
must := []interface{}{ must := []interface{}{}
}
if indexFilter, hasIndexPri := h.getAllowedIndexFilter(req); hasIndexPri { if indexFilter, hasIndexPri := h.getAllowedIndexFilter(req); hasIndexPri {
if indexFilter != nil{ if indexFilter != nil {
must = append(must, indexFilter) must = append(must, indexFilter)
} }
}else{ } else {
h.WriteJSON(w, elastic.SearchResponse{ h.WriteJSON(w, elastic.SearchResponse{}, http.StatusOK)
}, http.StatusOK)
return return
} }
boolQuery := util.MapStr{ boolQuery := util.MapStr{
@ -204,14 +202,14 @@ func (h *APIHandler) SearchIndexMetadata(w http.ResponseWriter, req *http.Reques
response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(orm.GetIndexName(elastic.IndexConfig{}), dsl) response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(orm.GetIndexName(elastic.IndexConfig{}), dsl)
if err != nil { if err != nil {
resBody["error"] = err.Error() resBody["error"] = err.Error()
h.WriteJSON(w,resBody, http.StatusInternalServerError ) h.WriteJSON(w, resBody, http.StatusInternalServerError)
return return
} }
w.Write(util.MustToJSONBytes(response)) w.Write(util.MustToJSONBytes(response))
} }
func (h *APIHandler) getAllowedIndexFilter(req *http.Request) (util.MapStr, bool){ func (h *APIHandler) getAllowedIndexFilter(req *http.Request) (util.MapStr, bool) {
hasAllPrivilege, indexPrivilege := h.GetCurrentUserIndex(req) hasAllPrivilege, indexPrivilege := h.GetCurrentUserIndex(req)
if !hasAllPrivilege && len(indexPrivilege) == 0 { if !hasAllPrivilege && len(indexPrivilege) == 0 {
return nil, false return nil, false
@ -224,7 +222,7 @@ func (h *APIHandler) getAllowedIndexFilter(req *http.Request) (util.MapStr, bool
normalIndices []string normalIndices []string
) )
for _, index := range indices { for _, index := range indices {
if strings.Contains(index,"*") { if strings.Contains(index, "*") {
wildcardIndices = append(wildcardIndices, index) wildcardIndices = append(wildcardIndices, index)
continue continue
} }
@ -318,7 +316,7 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, req *http.Request, p
return return
} }
clusterIndexNames[firstClusterID] = append(clusterIndexNames[firstClusterID], firstIndexName) clusterIndexNames[firstClusterID] = append(clusterIndexNames[firstClusterID], firstIndexName)
}else{ } else {
h.WriteError(w, fmt.Sprintf("invalid index_id: %v", indexID), http.StatusInternalServerError) h.WriteError(w, fmt.Sprintf("invalid index_id: %v", indexID), http.StatusInternalServerError)
return return
} }
@ -382,7 +380,7 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, req *http.Request, p
} }
if primary == true { if primary == true {
indexInfo.Shards++ indexInfo.Shards++
}else{ } else {
indexInfo.Replicas++ indexInfo.Replicas++
} }
indexInfo.Timestamp = hitM["timestamp"] indexInfo.Timestamp = hitM["timestamp"]
@ -403,11 +401,11 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, req *http.Request, p
} }
var metricLen = 15 var metricLen = 15
// 索引速率 // 索引速率
indexMetric:=newMetricItem("indexing", 1, OperationGroupKey) indexMetric := newMetricItem("indexing", 1, OperationGroupKey)
indexMetric.OnlyPrimary = true indexMetric.OnlyPrimary = true
indexMetric.AddAxi("indexing rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) indexMetric.AddAxi("indexing rate", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems := []GroupMetricItem{} nodeMetricItems := []GroupMetricItem{}
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "indexing", Key: "indexing",
Field: "payload.elasticsearch.shard_stats.indexing.index_total", Field: "payload.elasticsearch.shard_stats.indexing.index_total",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -416,9 +414,9 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, req *http.Request, p
FormatType: "num", FormatType: "num",
Units: "Indexing/s", Units: "Indexing/s",
}) })
queryMetric:=newMetricItem("search", 2, OperationGroupKey) queryMetric := newMetricItem("search", 2, OperationGroupKey)
queryMetric.AddAxi("query rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) queryMetric.AddAxi("query rate", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "search", Key: "search",
Field: "payload.elasticsearch.shard_stats.search.query_total", Field: "payload.elasticsearch.shard_stats.search.query_total",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -428,9 +426,9 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, req *http.Request, p
Units: "Search/s", Units: "Search/s",
}) })
aggs:=map[string]interface{}{} aggs := map[string]interface{}{}
query :=map[string]interface{}{} query := map[string]interface{}{}
query["query"]=util.MapStr{ query["query"] = util.MapStr{
"bool": util.MapStr{ "bool": util.MapStr{
"must": []util.MapStr{ "must": []util.MapStr{
{ {
@ -462,7 +460,7 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, req *http.Request, p
{ {
"range": util.MapStr{ "range": util.MapStr{
"timestamp": util.MapStr{ "timestamp": util.MapStr{
"gte": fmt.Sprintf("now-%ds", metricLen * bucketSize), "gte": fmt.Sprintf("now-%ds", metricLen*bucketSize),
}, },
}, },
}, },
@ -471,18 +469,18 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, req *http.Request, p
} }
sumAggs := util.MapStr{} sumAggs := util.MapStr{}
for _,metricItem:=range nodeMetricItems{ for _, metricItem := range nodeMetricItems {
leafAgg := util.MapStr{ leafAgg := util.MapStr{
"max":util.MapStr{ "max": util.MapStr{
"field": metricItem.Field, "field": metricItem.Field,
}, },
} }
var sumBucketPath = "term_shard>"+ metricItem.ID var sumBucketPath = "term_shard>" + metricItem.ID
if metricItem.MetricItem.OnlyPrimary { if metricItem.MetricItem.OnlyPrimary {
filterSubAggs := util.MapStr{ filterSubAggs := util.MapStr{
metricItem.ID: leafAgg, metricItem.ID: leafAgg,
} }
aggs["filter_pri"]=util.MapStr{ aggs["filter_pri"] = util.MapStr{
"filter": util.MapStr{ "filter": util.MapStr{
"term": util.MapStr{ "term": util.MapStr{
"payload.elasticsearch.shard_stats.routing.primary": util.MapStr{ "payload.elasticsearch.shard_stats.routing.primary": util.MapStr{
@ -492,8 +490,8 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, req *http.Request, p
}, },
"aggs": filterSubAggs, "aggs": filterSubAggs,
} }
sumBucketPath = "term_shard>filter_pri>"+ metricItem.ID sumBucketPath = "term_shard>filter_pri>" + metricItem.ID
}else{ } else {
aggs[metricItem.ID] = leafAgg aggs[metricItem.ID] = leafAgg
} }
@ -502,15 +500,15 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, req *http.Request, p
"buckets_path": sumBucketPath, "buckets_path": sumBucketPath,
}, },
} }
if metricItem.IsDerivative{ if metricItem.IsDerivative {
sumAggs[metricItem.ID+"_deriv"]=util.MapStr{ sumAggs[metricItem.ID+"_deriv"] = util.MapStr{
"derivative":util.MapStr{ "derivative": util.MapStr{
"buckets_path": metricItem.ID, "buckets_path": metricItem.ID,
}, },
} }
} }
} }
sumAggs["term_shard"]= util.MapStr{ sumAggs["term_shard"] = util.MapStr{
"terms": util.MapStr{ "terms": util.MapStr{
"field": "metadata.labels.shard_id", "field": "metadata.labels.shard_id",
"size": 10000, "size": 10000,
@ -523,8 +521,8 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, req *http.Request, p
if err != nil { if err != nil {
panic(err) panic(err)
} }
query["size"]=0 query["size"] = 0
query["aggs"]= util.MapStr{ query["aggs"] = util.MapStr{
"group_by_level": util.MapStr{ "group_by_level": util.MapStr{
"terms": util.MapStr{ "terms": util.MapStr{
"field": "metadata.labels.index_id", "field": "metadata.labels.index_id",
@ -532,11 +530,11 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, req *http.Request, p
}, },
"aggs": util.MapStr{ "aggs": util.MapStr{
"dates": util.MapStr{ "dates": util.MapStr{
"date_histogram":util.MapStr{ "date_histogram": util.MapStr{
"field": "timestamp", "field": "timestamp",
intervalField: bucketSizeStr, intervalField: bucketSizeStr,
}, },
"aggs":sumAggs, "aggs": sumAggs,
}, },
}, },
}, },
@ -549,9 +547,8 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, req *http.Request, p
indexMetrics := map[string]util.MapStr{} indexMetrics := map[string]util.MapStr{}
for key, item := range metrics { for key, item := range metrics {
for _, line := range item.Lines { for _, line := range item.Lines {
if _, ok := indexMetrics[line.Metric.Label]; !ok{ if _, ok := indexMetrics[line.Metric.Label]; !ok {
indexMetrics[line.Metric.Label] = util.MapStr{ indexMetrics[line.Metric.Label] = util.MapStr{}
}
} }
indexMetrics[line.Metric.Label][key] = line.Data indexMetrics[line.Metric.Label][key] = line.Data
} }
@ -605,7 +602,7 @@ func (h *APIHandler) GetIndexInfo(w http.ResponseWriter, req *http.Request, ps h
return return
} }
if len(parts) < 2 { if len(parts) < 2 {
h.WriteError(w, "invalid index id: "+ indexID, http.StatusInternalServerError) h.WriteError(w, "invalid index id: "+indexID, http.StatusInternalServerError)
return return
} }
@ -683,7 +680,7 @@ func (h *APIHandler) GetIndexInfo(w http.ResponseWriter, req *http.Request, ps h
storeInBytes, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "store", "size_in_bytes"}, resultM) storeInBytes, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "store", "size_in_bytes"}, resultM)
if docs, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "docs", "count"}, resultM); ok { if docs, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "docs", "count"}, resultM); ok {
//summary["docs"] = docs //summary["docs"] = docs
if v, ok := docs.(float64); ok && primary == true{ if v, ok := docs.(float64); ok && primary == true {
shardSum.DocsCount += int64(v) shardSum.DocsCount += int64(v)
} }
} }
@ -695,7 +692,7 @@ func (h *APIHandler) GetIndexInfo(w http.ResponseWriter, req *http.Request, ps h
} }
if primary == true { if primary == true {
shardSum.Shards++ shardSum.Shards++
}else{ } else {
shardSum.Replicas++ shardSum.Replicas++
} }
} }
@ -706,7 +703,7 @@ func (h *APIHandler) GetIndexInfo(w http.ResponseWriter, req *http.Request, ps h
indexInfo["store_size"] = util.FormatBytes(float64(shardSum.StoreInBytes), 1) indexInfo["store_size"] = util.FormatBytes(float64(shardSum.StoreInBytes), 1)
indexInfo["shards"] = shardSum.Shards + shardSum.Replicas indexInfo["shards"] = shardSum.Shards + shardSum.Replicas
summary["unassigned_shards"] = (replicasNum + 1) * shardsNum - shardSum.Shards - shardSum.Replicas summary["unassigned_shards"] = (replicasNum+1)*shardsNum - shardSum.Shards - shardSum.Replicas
} }
summary["index_info"] = indexInfo summary["index_info"] = indexInfo
@ -742,7 +739,7 @@ func (h *APIHandler) GetIndexShards(w http.ResponseWriter, req *http.Request, ps
err, result := orm.Search(&event.Event{}, &q1) err, result := orm.Search(&event.Event{}, &q1)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
h.WriteError(w,err.Error(), http.StatusInternalServerError ) h.WriteError(w, err.Error(), http.StatusInternalServerError)
return return
} }
var shards = []interface{}{} var shards = []interface{}{}
@ -756,7 +753,7 @@ func (h *APIHandler) GetIndexShards(w http.ResponseWriter, req *http.Request, ps
err, nodesResult := orm.Search(elastic.NodeConfig{}, q) err, nodesResult := orm.Search(elastic.NodeConfig{}, q)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
h.WriteError(w,err.Error(), http.StatusInternalServerError ) h.WriteError(w, err.Error(), http.StatusInternalServerError)
return return
} }
nodeIDToName := util.MapStr{} nodeIDToName := util.MapStr{}
@ -803,7 +800,7 @@ func (h *APIHandler) GetIndexShards(w http.ResponseWriter, req *http.Request, ps
primary, _ := shardM.GetValue("routing.primary") primary, _ := shardM.GetValue("routing.primary")
if primary == true { if primary == true {
shardInfo["prirep"] = "p" shardInfo["prirep"] = "p"
}else{ } else {
shardInfo["prirep"] = "r" shardInfo["prirep"] = "r"
} }
shardInfo["state"], _ = shardM.GetValue("routing.state") shardInfo["state"], _ = shardM.GetValue("routing.state")
@ -880,11 +877,11 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ
var metricType string var metricType string
if metricKey == v1.IndexHealthMetricKey { if metricKey == v1.IndexHealthMetricKey {
metricType = v1.MetricTypeClusterHealth metricType = v1.MetricTypeClusterHealth
}else{ } else {
//for agent mode //for agent mode
metricType = v1.MetricTypeNodeStats metricType = v1.MetricTypeNodeStats
} }
bucketSize, min, max, err := h.GetMetricRangeAndBucketSize(req, clusterID, metricType,60) bucketSize, min, max, err := h.GetMetricRangeAndBucketSize(req, clusterID, metricType, 60)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
resBody["error"] = err resBody["error"] = err
@ -892,7 +889,7 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ
return return
} }
if bucketSize <= 60 { if bucketSize <= 60 {
min = min - int64(2 * bucketSize * 1000) min = min - int64(2*bucketSize*1000)
} }
timeout := h.GetParameterOrDefault(req, "timeout", "60s") timeout := h.GetParameterOrDefault(req, "timeout", "60s")
du, err := time.ParseDuration(timeout) du, err := time.ParseDuration(timeout)
@ -947,7 +944,7 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ
return return
} }
metrics["shard_state"] = shardStateMetric metrics["shard_state"] = shardStateMetric
}else if metricKey == v1.IndexHealthMetricKey { } else if metricKey == v1.IndexHealthMetricKey {
healthMetric, err := h.GetIndexHealthMetric(ctx, clusterID, indexName, min, max, bucketSize) healthMetric, err := h.GetIndexHealthMetric(ctx, clusterID, indexName, min, max, bucketSize)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
@ -1037,7 +1034,7 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ
minBucketSize, err := v1.GetMetricMinBucketSize(clusterID, metricType) minBucketSize, err := v1.GetMetricMinBucketSize(clusterID, metricType)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
}else{ } else {
metrics[metricKey].MinBucketSize = int64(minBucketSize) metrics[metricKey].MinBucketSize = int64(minBucketSize)
} }
} }
@ -1047,8 +1044,8 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ
h.WriteJSON(w, resBody, http.StatusOK) h.WriteJSON(w, resBody, http.StatusOK)
} }
func (h *APIHandler) getIndexShardsMetric(ctx context.Context, id, indexName string, min, max int64, bucketSize int)(*common.MetricItem, error){ func (h *APIHandler) getIndexShardsMetric(ctx context.Context, id, indexName string, min, max int64, bucketSize int) (*common.MetricItem, error) {
bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr)
if err != nil { if err != nil {
return nil, err return nil, err
@ -1122,8 +1119,8 @@ func (h *APIHandler) getIndexShardsMetric(ctx context.Context, id, indexName str
return nil, err return nil, err
} }
metricItem:=newMetricItem("shard_state", 0, "") metricItem := newMetricItem("shard_state", 0, "")
metricItem.AddLine("Shard State","Shard State","","group1","payload.elasticsearch.shard_stats.routing.state","max",bucketSizeStr,"","ratio","0.[00]","0.[00]",false,false) metricItem.AddLine("Shard State", "Shard State", "", "group1", "payload.elasticsearch.shard_stats.routing.state", "max", bucketSizeStr, "", "ratio", "0.[00]", "0.[00]", false, false)
metricData := []interface{}{} metricData := []interface{}{}
if response.StatusCode == 200 { if response.StatusCode == 200 {
@ -1140,7 +1137,7 @@ func (h *APIHandler) getIndexShardsMetric(ctx context.Context, id, indexName str
} }
func (h *APIHandler) getIndexNodes(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *APIHandler) getIndexNodes(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody := map[string] interface{}{} resBody := map[string]interface{}{}
id := ps.ByName("id") id := ps.ByName("id")
indexName := ps.ByName("index") indexName := ps.ByName("index")
if !h.IsIndexAllowed(req, id, indexName) { if !h.IsIndexAllowed(req, id, indexName) {
@ -1149,7 +1146,7 @@ func (h *APIHandler) getIndexNodes(w http.ResponseWriter, req *http.Request, ps
}, http.StatusForbidden) }, http.StatusForbidden)
return return
} }
q := &orm.Query{ Size: 1} q := &orm.Query{Size: 1}
q.AddSort("timestamp", orm.DESC) q.AddSort("timestamp", orm.DESC)
q.Conds = orm.And( q.Conds = orm.And(
orm.Eq("metadata.category", "elasticsearch"), orm.Eq("metadata.category", "elasticsearch"),
@ -1161,13 +1158,13 @@ func (h *APIHandler) getIndexNodes(w http.ResponseWriter, req *http.Request, ps
err, result := orm.Search(event.Event{}, q) err, result := orm.Search(event.Event{}, q)
if err != nil { if err != nil {
resBody["error"] = err.Error() resBody["error"] = err.Error()
h.WriteJSON(w,resBody, http.StatusInternalServerError ) h.WriteJSON(w, resBody, http.StatusInternalServerError)
} }
namesM := util.MapStr{} namesM := util.MapStr{}
if len(result.Result) > 0 { if len(result.Result) > 0 {
if data, ok := result.Result[0].(map[string]interface{}); ok { if data, ok := result.Result[0].(map[string]interface{}); ok {
if routingTable, exists := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "index_routing_table"}, data); exists { if routingTable, exists := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "index_routing_table"}, data); exists {
if table, ok := routingTable.(map[string]interface{}); ok{ if table, ok := routingTable.(map[string]interface{}); ok {
if shardsM, ok := table["shards"].(map[string]interface{}); ok { if shardsM, ok := table["shards"].(map[string]interface{}); ok {
for _, rows := range shardsM { for _, rows := range shardsM {
if rowsArr, ok := rows.([]interface{}); ok { if rowsArr, ok := rows.([]interface{}); ok {
@ -1189,12 +1186,12 @@ func (h *APIHandler) getIndexNodes(w http.ResponseWriter, req *http.Request, ps
} }
//node uuid //node uuid
nodeIds := make([]interface{}, 0, len(namesM) ) nodeIds := make([]interface{}, 0, len(namesM))
for name, _ := range namesM { for name, _ := range namesM {
nodeIds = append(nodeIds, name) nodeIds = append(nodeIds, name)
} }
q1 := &orm.Query{ Size: 100} q1 := &orm.Query{Size: 100}
q1.AddSort("timestamp", orm.DESC) q1.AddSort("timestamp", orm.DESC)
q1.Conds = orm.And( q1.Conds = orm.And(
orm.Eq("metadata.category", "elasticsearch"), orm.Eq("metadata.category", "elasticsearch"),
@ -1204,7 +1201,7 @@ func (h *APIHandler) getIndexNodes(w http.ResponseWriter, req *http.Request, ps
err, result = orm.Search(elastic.NodeConfig{}, q1) err, result = orm.Search(elastic.NodeConfig{}, q1)
if err != nil { if err != nil {
resBody["error"] = err.Error() resBody["error"] = err.Error()
h.WriteJSON(w,resBody, http.StatusInternalServerError ) h.WriteJSON(w, resBody, http.StatusInternalServerError)
} }
nodes := []interface{}{} nodes := []interface{}{}
for _, hit := range result.Result { for _, hit := range result.Result {
@ -1249,7 +1246,7 @@ func (h APIHandler) ListIndex(w http.ResponseWriter, req *http.Request, ps httpr
} }
var must = []util.MapStr{} var must = []util.MapStr{}
if !util.StringInArray(ids, "*"){ if !util.StringInArray(ids, "*") {
must = append(must, util.MapStr{ must = append(must, util.MapStr{
"terms": util.MapStr{ "terms": util.MapStr{
@ -1260,9 +1257,8 @@ func (h APIHandler) ListIndex(w http.ResponseWriter, req *http.Request, ps httpr
if keyword != "" { if keyword != "" {
must = append(must, util.MapStr{ must = append(must, util.MapStr{
"wildcard":util.MapStr{ "wildcard": util.MapStr{
"metadata.index_name": "metadata.index_name": util.MapStr{"value": fmt.Sprintf("*%s*", keyword)},
util.MapStr{"value": fmt.Sprintf("*%s*", keyword)},
}, },
}) })
} }
@ -1288,7 +1284,6 @@ func (h APIHandler) ListIndex(w http.ResponseWriter, req *http.Request, ps httpr
}, },
} }
esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
indexName := orm.GetIndexName(elastic.IndexConfig{}) indexName := orm.GetIndexName(elastic.IndexConfig{})
resp, err := esClient.SearchWithRawQueryDSL(indexName, util.MustToJSONBytes(dsl)) resp, err := esClient.SearchWithRawQueryDSL(indexName, util.MustToJSONBytes(dsl))
@ -1310,7 +1305,7 @@ func (h APIHandler) ListIndex(w http.ResponseWriter, req *http.Request, ps httpr
return return
} }
//deleteIndexMetadata used to delete index metadata after index is deleted from cluster // deleteIndexMetadata used to delete index metadata after index is deleted from cluster
func (h APIHandler) deleteIndexMetadata(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h APIHandler) deleteIndexMetadata(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
indexName := orm.GetIndexName(elastic.IndexConfig{}) indexName := orm.GetIndexName(elastic.IndexConfig{})
@ -1325,7 +1320,7 @@ func (h APIHandler) deleteIndexMetadata(w http.ResponseWriter, req *http.Request
if indexFilter != nil { if indexFilter != nil {
must = append(must, indexFilter) must = append(must, indexFilter)
} }
}else{ } else {
//has no any index permission, just return //has no any index permission, just return
h.WriteAckOKJSON(w) h.WriteAckOKJSON(w)
return return

27
modules/elastic/api/manage.go
View File

@ -27,6 +27,13 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"math"
"net/http"
"strconv"
"strings"
"sync"
"time"
log "github.com/cihub/seelog" log "github.com/cihub/seelog"
"infini.sh/console/core" "infini.sh/console/core"
v1 "infini.sh/console/modules/elastic/api/v1" v1 "infini.sh/console/modules/elastic/api/v1"
@ -39,12 +46,6 @@ import (
"infini.sh/framework/core/orm" "infini.sh/framework/core/orm"
"infini.sh/framework/core/util" "infini.sh/framework/core/util"
"infini.sh/framework/modules/elastic/common" "infini.sh/framework/modules/elastic/common"
"math"
"net/http"
"strconv"
"strings"
"sync"
"time"
) )
type APIHandler struct { type APIHandler struct {
@ -534,7 +535,7 @@ func (h *APIHandler) HandleClusterMetricsAction(w http.ResponseWriter, req *http
key := h.GetParameter(req, "key") key := h.GetParameter(req, "key")
var metricType string var metricType string
switch key { switch key {
case v1.IndexThroughputMetricKey, v1.SearchThroughputMetricKey, v1.IndexLatencyMetricKey, v1.SearchLatencyMetricKey, CircuitBreakerMetricKey,ShardStateMetricKey: case v1.IndexThroughputMetricKey, v1.SearchThroughputMetricKey, v1.IndexLatencyMetricKey, v1.SearchLatencyMetricKey, CircuitBreakerMetricKey, ShardStateMetricKey:
metricType = v1.MetricTypeNodeStats metricType = v1.MetricTypeNodeStats
case ClusterDocumentsMetricKey, case ClusterDocumentsMetricKey,
ClusterStorageMetricKey, ClusterStorageMetricKey,
@ -649,7 +650,7 @@ func (h *APIHandler) HandleClusterMetricsAction(w http.ResponseWriter, req *http
minBucketSize, err := v1.GetMetricMinBucketSize(id, metricType) minBucketSize, err := v1.GetMetricMinBucketSize(id, metricType)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
}else{ } else {
metrics[key].MinBucketSize = int64(minBucketSize) metrics[key].MinBucketSize = int64(minBucketSize)
} }
} }
@ -700,7 +701,7 @@ func (h *APIHandler) HandleNodeMetricsAction(w http.ResponseWriter, req *http.Re
minBucketSize, err := v1.GetMetricMinBucketSize(id, v1.MetricTypeNodeStats) minBucketSize, err := v1.GetMetricMinBucketSize(id, v1.MetricTypeNodeStats)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
}else{ } else {
metrics[key].MinBucketSize = int64(minBucketSize) metrics[key].MinBucketSize = int64(minBucketSize)
} }
} }
@ -817,7 +818,7 @@ func (h *APIHandler) HandleIndexMetricsAction(w http.ResponseWriter, req *http.R
} }
} }
}else{ } else {
metrics, err = h.getIndexMetrics(ctx, req, id, bucketSize, min, max, indexName, top, shardID, key) metrics, err = h.getIndexMetrics(ctx, req, id, bucketSize, min, max, indexName, top, shardID, key)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
@ -830,7 +831,7 @@ func (h *APIHandler) HandleIndexMetricsAction(w http.ResponseWriter, req *http.R
minBucketSize, err := v1.GetMetricMinBucketSize(id, v1.MetricTypeNodeStats) minBucketSize, err := v1.GetMetricMinBucketSize(id, v1.MetricTypeNodeStats)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
}else{ } else {
metrics[key].MinBucketSize = int64(minBucketSize) metrics[key].MinBucketSize = int64(minBucketSize)
} }
} }
@ -888,7 +889,7 @@ func (h *APIHandler) HandleQueueMetricsAction(w http.ResponseWriter, req *http.R
minBucketSize, err := v1.GetMetricMinBucketSize(id, v1.MetricTypeNodeStats) minBucketSize, err := v1.GetMetricMinBucketSize(id, v1.MetricTypeNodeStats)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
}else{ } else {
metrics[key].MinBucketSize = int64(minBucketSize) metrics[key].MinBucketSize = int64(minBucketSize)
} }
} }
@ -1027,7 +1028,7 @@ const (
func (h *APIHandler) GetClusterMetrics(ctx context.Context, id string, bucketSize int, min, max int64, metricKey string) (map[string]*common.MetricItem, error) { func (h *APIHandler) GetClusterMetrics(ctx context.Context, id string, bucketSize int, min, max int64, metricKey string) (map[string]*common.MetricItem, error) {
var ( var (
clusterMetricsResult = map[string]*common.MetricItem {} clusterMetricsResult = map[string]*common.MetricItem{}
err error err error
) )
switch metricKey { switch metricKey {

38
modules/elastic/api/metrics_util.go
View File

@ -112,7 +112,7 @@ func generateGroupAggs(nodeMetricItems []GroupMetricItem) map[string]interface{}
func (h *APIHandler) getMetrics(ctx context.Context, query map[string]interface{}, grpMetricItems []GroupMetricItem, bucketSize int) (map[string]*common.MetricItem, error) { func (h *APIHandler) getMetrics(ctx context.Context, query map[string]interface{}, grpMetricItems []GroupMetricItem, bucketSize int) (map[string]*common.MetricItem, error) {
bucketSizeStr := fmt.Sprintf("%vs", bucketSize) bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
queryDSL := util.MustToJSONBytes(query) queryDSL := util.MustToJSONBytes(query)
response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).QueryDSL(ctx, getAllMetricsIndex(),nil, queryDSL) response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).QueryDSL(ctx, getAllMetricsIndex(), nil, queryDSL)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -205,12 +205,12 @@ func (h *APIHandler) getMetrics(ctx context.Context, query map[string]interface{
dataKey = dataKey + "_deriv" dataKey = dataKey + "_deriv"
} }
line.Data = grpMetricData[dataKey][line.Metric.Label] line.Data = grpMetricData[dataKey][line.Metric.Label]
if v, ok := line.Data.([][]interface{}); ok && len(v)> 0 && bucketSize <= 60 { if v, ok := line.Data.([][]interface{}); ok && len(v) > 0 && bucketSize <= 60 {
// remove first metric dot // remove first metric dot
temp := v[1:] temp := v[1:]
// // remove first last dot // // remove first last dot
if len(temp) > 0 { if len(temp) > 0 {
temp = temp[0: len(temp)-1] temp = temp[0 : len(temp)-1]
} }
line.Data = temp line.Data = temp
} }
@ -429,12 +429,12 @@ func (h *APIHandler) getSingleMetrics(ctx context.Context, metricItems []*common
for _, line := range metricItem.Lines { for _, line := range metricItem.Lines {
line.TimeRange = common.TimeRange{Min: minDate, Max: maxDate} line.TimeRange = common.TimeRange{Min: minDate, Max: maxDate}
line.Data = metricData[line.Metric.GetDataKey()] line.Data = metricData[line.Metric.GetDataKey()]
if v, ok := line.Data.([][]interface{}); ok && len(v)> 0 && bucketSize <= 60 { if v, ok := line.Data.([][]interface{}); ok && len(v) > 0 && bucketSize <= 60 {
// remove first metric dot // remove first metric dot
temp := v[1:] temp := v[1:]
// // remove first last dot // // remove first last dot
if len(temp) > 0 { if len(temp) > 0 {
temp = temp[0: len(temp)-1] temp = temp[0 : len(temp)-1]
} }
line.Data = temp line.Data = temp
} }
@ -912,13 +912,13 @@ func parseGroupMetricData(buckets []elastic.BucketBase, isPercent bool) ([]inter
if bkMap, ok := statusBk.(map[string]interface{}); ok { if bkMap, ok := statusBk.(map[string]interface{}); ok {
statusKey := bkMap["key"].(string) statusKey := bkMap["key"].(string)
count := bkMap["doc_count"].(float64) count := bkMap["doc_count"].(float64)
if isPercent{ if isPercent {
metricData = append(metricData, map[string]interface{}{ metricData = append(metricData, map[string]interface{}{
"x": dateTime, "x": dateTime,
"y": count / totalCount * 100, "y": count / totalCount * 100,
"g": statusKey, "g": statusKey,
}) })
}else{ } else {
metricData = append(metricData, map[string]interface{}{ metricData = append(metricData, map[string]interface{}{
"x": dateTime, "x": dateTime,
"y": count, "y": count,
@ -950,7 +950,7 @@ func (h *APIHandler) getSingleIndexMetricsByNodeStats(ctx context.Context, metri
"field": line.Metric.Field, "field": line.Metric.Field,
}, },
} }
var sumBucketPath = "term_node>"+ line.Metric.ID var sumBucketPath = "term_node>" + line.Metric.ID
aggs[line.Metric.ID] = leafAgg aggs[line.Metric.ID] = leafAgg
sumAggs[line.Metric.ID] = util.MapStr{ sumAggs[line.Metric.ID] = util.MapStr{
@ -966,9 +966,9 @@ func (h *APIHandler) getSingleIndexMetricsByNodeStats(ctx context.Context, metri
} }
aggs[line.Metric.ID+"_field2"] = leafAgg2 aggs[line.Metric.ID+"_field2"] = leafAgg2
sumAggs[line.Metric.ID + "_field2"] = util.MapStr{ sumAggs[line.Metric.ID+"_field2"] = util.MapStr{
"sum_bucket": util.MapStr{ "sum_bucket": util.MapStr{
"buckets_path": sumBucketPath+"_field2", "buckets_path": sumBucketPath + "_field2",
}, },
} }
} }
@ -991,7 +991,7 @@ func (h *APIHandler) getSingleIndexMetricsByNodeStats(ctx context.Context, metri
} }
} }
sumAggs["term_node"]= util.MapStr{ sumAggs["term_node"] = util.MapStr{
"terms": util.MapStr{ "terms": util.MapStr{
"field": "metadata.labels.node_id", "field": "metadata.labels.node_id",
"size": 1000, "size": 1000,
@ -1015,7 +1015,7 @@ func (h *APIHandler) getSingleIndexMetricsByNodeStats(ctx context.Context, metri
"aggs": sumAggs, "aggs": sumAggs,
}, },
} }
return parseSingleIndexMetrics(ctx, clusterID, metricItems, query, bucketSize,metricData, metricItemsMap) return parseSingleIndexMetrics(ctx, clusterID, metricItems, query, bucketSize, metricData, metricItemsMap)
} }
func (h *APIHandler) getSingleIndexMetrics(ctx context.Context, metricItems []*common.MetricItem, query map[string]interface{}, bucketSize int) (map[string]*common.MetricItem, error) { func (h *APIHandler) getSingleIndexMetrics(ctx context.Context, metricItems []*common.MetricItem, query map[string]interface{}, bucketSize int) (map[string]*common.MetricItem, error) {
@ -1035,7 +1035,7 @@ func (h *APIHandler) getSingleIndexMetrics(ctx context.Context, metricItems []*c
"field": line.Metric.Field, "field": line.Metric.Field,
}, },
} }
var sumBucketPath = "term_shard>"+ line.Metric.ID var sumBucketPath = "term_shard>" + line.Metric.ID
aggs[line.Metric.ID] = leafAgg aggs[line.Metric.ID] = leafAgg
sumAggs[line.Metric.ID] = util.MapStr{ sumAggs[line.Metric.ID] = util.MapStr{
"sum_bucket": util.MapStr{ "sum_bucket": util.MapStr{
@ -1050,9 +1050,9 @@ func (h *APIHandler) getSingleIndexMetrics(ctx context.Context, metricItems []*c
} }
aggs[line.Metric.ID+"_field2"] = leafAgg2 aggs[line.Metric.ID+"_field2"] = leafAgg2
sumAggs[line.Metric.ID + "_field2"] = util.MapStr{ sumAggs[line.Metric.ID+"_field2"] = util.MapStr{
"sum_bucket": util.MapStr{ "sum_bucket": util.MapStr{
"buckets_path": sumBucketPath+"_field2", "buckets_path": sumBucketPath + "_field2",
}, },
} }
} }
@ -1075,7 +1075,7 @@ func (h *APIHandler) getSingleIndexMetrics(ctx context.Context, metricItems []*c
} }
} }
sumAggs["term_shard"]= util.MapStr{ sumAggs["term_shard"] = util.MapStr{
"terms": util.MapStr{ "terms": util.MapStr{
"field": "metadata.labels.shard_id", "field": "metadata.labels.shard_id",
"size": 100000, "size": 100000,
@ -1109,7 +1109,7 @@ func (h *APIHandler) getSingleIndexMetrics(ctx context.Context, metricItems []*c
"aggs": sumAggs, "aggs": sumAggs,
}, },
} }
return parseSingleIndexMetrics(ctx, clusterID, metricItems, query, bucketSize,metricData, metricItemsMap) return parseSingleIndexMetrics(ctx, clusterID, metricItems, query, bucketSize, metricData, metricItemsMap)
} }
func parseSingleIndexMetrics(ctx context.Context, clusterID string, metricItems []*common.MetricItem, query map[string]interface{}, bucketSize int, metricData map[string][][]interface{}, metricItemsMap map[string]*common.MetricLine) (map[string]*common.MetricItem, error) { func parseSingleIndexMetrics(ctx context.Context, clusterID string, metricItems []*common.MetricItem, query map[string]interface{}, bucketSize int, metricData map[string][][]interface{}, metricItemsMap map[string]*common.MetricLine) (map[string]*common.MetricItem, error) {
@ -1174,12 +1174,12 @@ func parseSingleIndexMetrics(ctx context.Context, clusterID string, metricItems
for _, line := range metricItem.Lines { for _, line := range metricItem.Lines {
line.TimeRange = common.TimeRange{Min: minDate, Max: maxDate} line.TimeRange = common.TimeRange{Min: minDate, Max: maxDate}
line.Data = metricData[line.Metric.GetDataKey()] line.Data = metricData[line.Metric.GetDataKey()]
if v, ok := line.Data.([][]interface{}); ok && len(v)> 0 && bucketSize <= 60 { if v, ok := line.Data.([][]interface{}); ok && len(v) > 0 && bucketSize <= 60 {
// remove first metric dot // remove first metric dot
temp := v[1:] temp := v[1:]
// // remove first last dot // // remove first last dot
if len(temp) > 0 { if len(temp) > 0 {
temp = temp[0: len(temp)-1] temp = temp[0 : len(temp)-1]
} }
line.Data = temp line.Data = temp
} }

96
modules/elastic/api/metrics_util_test.go
View File

@ -33,83 +33,81 @@ import (
) )
func TestGetMetricParams(t *testing.T) { func TestGetMetricParams(t *testing.T) {
handler:=APIHandler{} handler := APIHandler{}
req, err :=http.NewRequest("GET","https://infinilabs.com/api/?bucket_size=1m",nil) req, err := http.NewRequest("GET", "https://infinilabs.com/api/?bucket_size=1m", nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
bucketSize, min, max, err:=handler.GetMetricRangeAndBucketSize(req,"", "",15) bucketSize, min, max, err := handler.GetMetricRangeAndBucketSize(req, "", "", 15)
fmt.Println(bucketSize) fmt.Println(bucketSize)
fmt.Println(util.FormatUnixTimestamp(min/1000))//2022-01-27 15:28:57 fmt.Println(util.FormatUnixTimestamp(min / 1000)) //2022-01-27 15:28:57
fmt.Println(util.FormatUnixTimestamp(max/1000))//2022-01-27 15:28:57 fmt.Println(util.FormatUnixTimestamp(max / 1000)) //2022-01-27 15:28:57
fmt.Println(time.Now())//2022-01-27 15:28:57 fmt.Println(time.Now()) //2022-01-27 15:28:57
fmt.Println(bucketSize, min, max, err) fmt.Println(bucketSize, min, max, err)
} }
func TestConvertBucketItemsToAggQueryParams(t *testing.T) { func TestConvertBucketItemsToAggQueryParams(t *testing.T) {
bucketItem:=common.BucketItem{} bucketItem := common.BucketItem{}
bucketItem.Key="key1" bucketItem.Key = "key1"
bucketItem.Type=common.TermsBucket bucketItem.Type = common.TermsBucket
bucketItem.Parameters=map[string]interface{}{} bucketItem.Parameters = map[string]interface{}{}
bucketItem.Parameters["field"]="metadata.labels.cluster_id" bucketItem.Parameters["field"] = "metadata.labels.cluster_id"
bucketItem.Parameters["size"]=2 bucketItem.Parameters["size"] = 2
nestBucket := common.BucketItem{}
nestBucket.Key = "key2"
nestBucket.Type = common.DateHistogramBucket
nestBucket.Parameters = map[string]interface{}{}
nestBucket.Parameters["field"] = "timestamp"
nestBucket.Parameters["calendar_interval"] = "1d"
nestBucket.Parameters["time_zone"] = "+08:00"
nestBucket:=common.BucketItem{} leafBucket := common.NewBucketItem(common.TermsBucket, util.MapStr{
nestBucket.Key="key2" "size": 5,
nestBucket.Type=common.DateHistogramBucket "field": "payload.elasticsearch.cluster_health.status",
nestBucket.Parameters=map[string]interface{}{}
nestBucket.Parameters["field"]="timestamp"
nestBucket.Parameters["calendar_interval"]="1d"
nestBucket.Parameters["time_zone"]="+08:00"
leafBucket:=common.NewBucketItem(common.TermsBucket,util.MapStr{
"size":5,
"field":"payload.elasticsearch.cluster_health.status",
}) })
leafBucket.Key="key3" leafBucket.Key = "key3"
metricItems:=[]*common.MetricItem{} metricItems := []*common.MetricItem{}
var bucketSizeStr ="10s" var bucketSizeStr = "10s"
metricItem:=newMetricItem("cluster_summary", 2, "cluster") metricItem := newMetricItem("cluster_summary", 2, "cluster")
metricItem.Key="key4" metricItem.Key = "key4"
metricItem.AddLine("Indexing","Total Indexing","Number of documents being indexed for primary and replica shards.","group1", metricItem.AddLine("Indexing", "Total Indexing", "Number of documents being indexed for primary and replica shards.", "group1",
"payload.elasticsearch.index_stats.total.indexing.index_total","max",bucketSizeStr,"doc/s","num","0,0.[00]","0,0.[00]",false,true) "payload.elasticsearch.index_stats.total.indexing.index_total", "max", bucketSizeStr, "doc/s", "num", "0,0.[00]", "0,0.[00]", false, true)
metricItem.AddLine("Search","Total Search","Number of search requests being executed across primary and replica shards. A single search can run against multiple shards!","group1", metricItem.AddLine("Search", "Total Search", "Number of search requests being executed across primary and replica shards. A single search can run against multiple shards!", "group1",
"payload.elasticsearch.index_stats.total.search.query_total","max",bucketSizeStr,"query/s","num","0,0.[00]","0,0.[00]",false,true) "payload.elasticsearch.index_stats.total.search.query_total", "max", bucketSizeStr, "query/s", "num", "0,0.[00]", "0,0.[00]", false, true)
metricItems=append(metricItems,metricItem) metricItems = append(metricItems, metricItem)
nestBucket.AddNestBucket(leafBucket) nestBucket.AddNestBucket(leafBucket)
nestBucket.Metrics=metricItems nestBucket.Metrics = metricItems
bucketItem.Buckets=[]*common.BucketItem{} bucketItem.Buckets = []*common.BucketItem{}
bucketItem.Buckets=append(bucketItem.Buckets,&nestBucket) bucketItem.Buckets = append(bucketItem.Buckets, &nestBucket)
aggs := ConvertBucketItemsToAggQuery([]*common.BucketItem{&bucketItem}, nil)
aggs:=ConvertBucketItemsToAggQuery([]*common.BucketItem{&bucketItem},nil)
fmt.Println(util.MustToJSON(aggs)) fmt.Println(util.MustToJSON(aggs))
response:="{ \"took\": 37, \"timed_out\": false, \"_shards\": { \"total\": 1, \"successful\": 1, \"skipped\": 0, \"failed\": 0 }, \"hits\": { \"total\": { \"value\": 10000, \"relation\": \"gte\" }, \"max_score\": null, \"hits\": [] }, \"aggregations\": { \"key1\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [ { \"key\": \"c7pqhptj69a0sg3rn05g\", \"doc_count\": 80482, \"key2\": { \"buckets\": [ { \"key_as_string\": \"2022-01-28T00:00:00.000+08:00\", \"key\": 1643299200000, \"doc_count\": 14310, \"c7qi5hii4h935v9bs91g\": { \"value\": 15680 }, \"key3\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [] }, \"c7qi5hii4h935v9bs920\": { \"value\": 2985 } }, { \"key_as_string\": \"2022-01-29T00:00:00.000+08:00\", \"key\": 1643385600000, \"doc_count\": 66172, \"c7qi5hii4h935v9bs91g\": { \"value\": 106206 }, \"key3\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [] }, \"c7qi5hii4h935v9bs920\": { \"value\": 20204 }, \"c7qi5hii4h935v9bs91g_deriv\": { \"value\": 90526 }, \"c7qi5hii4h935v9bs920_deriv\": { \"value\": 17219 } } ] } }, { \"key\": \"c7qi42ai4h92sksk979g\", \"doc_count\": 660, \"key2\": { \"buckets\": [ { \"key_as_string\": \"2022-01-29T00:00:00.000+08:00\", \"key\": 1643385600000, \"doc_count\": 660, \"c7qi5hii4h935v9bs91g\": { \"value\": 106206 }, \"key3\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [] }, \"c7qi5hii4h935v9bs920\": { \"value\": 20204 } } ] } } ] } } }" response := "{ \"took\": 37, \"timed_out\": false, \"_shards\": { \"total\": 1, \"successful\": 1, \"skipped\": 0, \"failed\": 0 }, \"hits\": { \"total\": { \"value\": 10000, \"relation\": \"gte\" }, \"max_score\": null, \"hits\": [] }, \"aggregations\": { \"key1\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [ { \"key\": \"c7pqhptj69a0sg3rn05g\", \"doc_count\": 80482, \"key2\": { \"buckets\": [ { \"key_as_string\": \"2022-01-28T00:00:00.000+08:00\", \"key\": 1643299200000, \"doc_count\": 14310, \"c7qi5hii4h935v9bs91g\": { \"value\": 15680 }, \"key3\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [] }, \"c7qi5hii4h935v9bs920\": { \"value\": 2985 } }, { \"key_as_string\": \"2022-01-29T00:00:00.000+08:00\", \"key\": 1643385600000, \"doc_count\": 66172, \"c7qi5hii4h935v9bs91g\": { \"value\": 106206 }, \"key3\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [] }, \"c7qi5hii4h935v9bs920\": { \"value\": 20204 }, \"c7qi5hii4h935v9bs91g_deriv\": { \"value\": 90526 }, \"c7qi5hii4h935v9bs920_deriv\": { \"value\": 17219 } } ] } }, { \"key\": \"c7qi42ai4h92sksk979g\", \"doc_count\": 660, \"key2\": { \"buckets\": [ { \"key_as_string\": \"2022-01-29T00:00:00.000+08:00\", \"key\": 1643385600000, \"doc_count\": 660, \"c7qi5hii4h935v9bs91g\": { \"value\": 106206 }, \"key3\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [] }, \"c7qi5hii4h935v9bs920\": { \"value\": 20204 } } ] } } ] } } }"
res:=SearchResponse{} res := SearchResponse{}
util.FromJSONBytes([]byte(response),&res) util.FromJSONBytes([]byte(response), &res)
fmt.Println(response) fmt.Println(response)
groupKey:="key1" groupKey := "key1"
metricLabelKey:="key2" metricLabelKey := "key2"
metricValueKey:="c7qi5hii4h935v9bs920" metricValueKey := "c7qi5hii4h935v9bs920"
data:=ParseAggregationResult(int(10),res.Aggregations,groupKey,metricLabelKey,metricValueKey) data := ParseAggregationResult(int(10), res.Aggregations, groupKey, metricLabelKey, metricValueKey)
fmt.Println(data) fmt.Println(data)
} }
func TestConvertBucketItems(t *testing.T) { func TestConvertBucketItems(t *testing.T) {
response:="{ \"took\": 8, \"timed_out\": false, \"_shards\": { \"total\": 1, \"successful\": 1, \"skipped\": 0, \"failed\": 0 }, \"hits\": { \"total\": { \"value\": 81, \"relation\": \"eq\" }, \"max_score\": null, \"hits\": [] }, \"aggregations\": { \"c7v2gm3i7638vvo4pv80\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [ { \"key\": \"c7uv7p3i76360kgdmpb0\", \"doc_count\": 81, \"c7v2gm3i7638vvo4pv8g\": { \"buckets\": [ { \"key_as_string\": \"2022-02-05T00:00:00.000+08:00\", \"key\": 1643990400000, \"doc_count\": 81, \"c7v2gm3i7638vvo4pv90\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [ { \"key\": \"yellow\", \"doc_count\": 81 } ] } } ] } } ] } } }" response := "{ \"took\": 8, \"timed_out\": false, \"_shards\": { \"total\": 1, \"successful\": 1, \"skipped\": 0, \"failed\": 0 }, \"hits\": { \"total\": { \"value\": 81, \"relation\": \"eq\" }, \"max_score\": null, \"hits\": [] }, \"aggregations\": { \"c7v2gm3i7638vvo4pv80\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [ { \"key\": \"c7uv7p3i76360kgdmpb0\", \"doc_count\": 81, \"c7v2gm3i7638vvo4pv8g\": { \"buckets\": [ { \"key_as_string\": \"2022-02-05T00:00:00.000+08:00\", \"key\": 1643990400000, \"doc_count\": 81, \"c7v2gm3i7638vvo4pv90\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [ { \"key\": \"yellow\", \"doc_count\": 81 } ] } } ] } } ] } } }"
res:=SearchResponse{} res := SearchResponse{}
util.FromJSONBytes([]byte(response),&res) util.FromJSONBytes([]byte(response), &res)
data:=ParseAggregationBucketResult(int(10),res.Aggregations,"c7v2gm3i7638vvo4pv80","c7v2gm3i7638vvo4pv8g","c7v2gm3i7638vvo4pv90", func() { data := ParseAggregationBucketResult(int(10), res.Aggregations, "c7v2gm3i7638vvo4pv80", "c7v2gm3i7638vvo4pv8g", "c7v2gm3i7638vvo4pv90", func() {
}) })

413
modules/elastic/api/node_metrics.go
View File

@ -107,8 +107,8 @@ const (
ModelInferenceBreakerMetricKey = "model_inference_breaker" ModelInferenceBreakerMetricKey = "model_inference_breaker"
) )
func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucketSize int, min, max int64, nodeName string, top int, metricKey string) (map[string]*common.MetricItem, error){ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucketSize int, min, max int64, nodeName string, top int, metricKey string) (map[string]*common.MetricItem, error) {
bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
clusterUUID, err := h.getClusterUUID(clusterID) clusterUUID, err := h.getClusterUUID(clusterID)
if err != nil { if err != nil {
return nil, err return nil, err
@ -158,7 +158,7 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
if nodeName != "" { if nodeName != "" {
nodeNames = strings.Split(nodeName, ",") nodeNames = strings.Split(nodeName, ",")
top = len(nodeNames) top = len(nodeNames)
}else{ } else {
nodeNames, err = h.getTopNodeName(clusterID, top, 15) nodeNames, err = h.getTopNodeName(clusterID, top, 15)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
@ -181,12 +181,11 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}, },
}, },
}, },
}) })
} }
query:=map[string]interface{}{} query := map[string]interface{}{}
query["query"]=util.MapStr{ query["query"] = util.MapStr{
"bool": util.MapStr{ "bool": util.MapStr{
"must": must, "must": must,
"filter": []util.MapStr{ "filter": []util.MapStr{
@ -205,8 +204,8 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
switch metricKey { switch metricKey {
case NodeProcessCPUMetricKey: case NodeProcessCPUMetricKey:
cpuMetric := newMetricItem(NodeProcessCPUMetricKey, 1, SystemGroupKey) cpuMetric := newMetricItem(NodeProcessCPUMetricKey, 1, SystemGroupKey)
cpuMetric.AddAxi("cpu","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true) cpuMetric.AddAxi("cpu", "group1", common.PositionLeft, "ratio", "0.[0]", "0.[0]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "cpu", Key: "cpu",
Field: "payload.elasticsearch.node_stats.process.cpu.percent", Field: "payload.elasticsearch.node_stats.process.cpu.percent",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -217,8 +216,8 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case NodeOSCPUMetricKey: case NodeOSCPUMetricKey:
osCpuMetric := newMetricItem(NodeOSCPUMetricKey, 2, SystemGroupKey) osCpuMetric := newMetricItem(NodeOSCPUMetricKey, 2, SystemGroupKey)
osCpuMetric.AddAxi("OS CPU Percent","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true) osCpuMetric.AddAxi("OS CPU Percent", "group1", common.PositionLeft, "ratio", "0.[0]", "0.[0]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "os_cpu", Key: "os_cpu",
Field: "payload.elasticsearch.node_stats.os.cpu.percent", Field: "payload.elasticsearch.node_stats.os.cpu.percent",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -229,8 +228,8 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case OSUsedMemoryMetricKey: case OSUsedMemoryMetricKey:
osMemMetric := newMetricItem(OSUsedMemoryMetricKey, 2, SystemGroupKey) osMemMetric := newMetricItem(OSUsedMemoryMetricKey, 2, SystemGroupKey)
osMemMetric.AddAxi("OS Mem Used Percent","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true) osMemMetric.AddAxi("OS Mem Used Percent", "group1", common.PositionLeft, "ratio", "0.[0]", "0.[0]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "os_used_mem", Key: "os_used_mem",
Field: "payload.elasticsearch.node_stats.os.mem.used_percent", Field: "payload.elasticsearch.node_stats.os.mem.used_percent",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -241,8 +240,8 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case OSLoadAverage1mMetricKey: case OSLoadAverage1mMetricKey:
osLoadMetric := newMetricItem(OSLoadAverage1mMetricKey, 2, SystemGroupKey) osLoadMetric := newMetricItem(OSLoadAverage1mMetricKey, 2, SystemGroupKey)
osLoadMetric.AddAxi("OS Load 1m Average","group1",common.PositionLeft,"","0.[0]","0.[0]",5,true) osLoadMetric.AddAxi("OS Load 1m Average", "group1", common.PositionLeft, "", "0.[0]", "0.[0]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "os_load_average_1m", Key: "os_load_average_1m",
Field: "payload.elasticsearch.node_stats.os.cpu.load_average.1m", Field: "payload.elasticsearch.node_stats.os.cpu.load_average.1m",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -254,15 +253,15 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
case OSUsedSwapMetricKey: case OSUsedSwapMetricKey:
//swap usage //swap usage
osSwapMetric := newMetricItem(OSUsedSwapMetricKey, 3, SystemGroupKey) osSwapMetric := newMetricItem(OSUsedSwapMetricKey, 3, SystemGroupKey)
osSwapMetric.AddAxi("OS Swap Used Percent","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true) osSwapMetric.AddAxi("OS Swap Used Percent", "group1", common.PositionLeft, "ratio", "0.[0]", "0.[0]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "os_used_swap", Key: "os_used_swap",
Field: "payload.elasticsearch.node_stats.os.swap.used_in_bytes", Field: "payload.elasticsearch.node_stats.os.swap.used_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
IsDerivative: false, IsDerivative: false,
Field2: "payload.elasticsearch.node_stats.os.swap.total_in_bytes", Field2: "payload.elasticsearch.node_stats.os.swap.total_in_bytes",
Calc: func(value, value2 float64) float64 { Calc: func(value, value2 float64) float64 {
return util.ToFixed((value / value2)*100, 2) return util.ToFixed((value/value2)*100, 2)
}, },
MetricItem: osSwapMetric, MetricItem: osSwapMetric,
FormatType: "ratio", FormatType: "ratio",
@ -270,8 +269,8 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case OpenFileMetricKey: case OpenFileMetricKey:
openFileMetric := newMetricItem(OpenFileMetricKey, 2, SystemGroupKey) openFileMetric := newMetricItem(OpenFileMetricKey, 2, SystemGroupKey)
openFileMetric.AddAxi("Open File Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) openFileMetric.AddAxi("Open File Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "open_file", Key: "open_file",
Field: "payload.elasticsearch.node_stats.process.open_file_descriptors", Field: "payload.elasticsearch.node_stats.process.open_file_descriptors",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -282,8 +281,8 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case OpenFilePercentMetricKey: case OpenFilePercentMetricKey:
openFilePercentMetric := newMetricItem(OpenFilePercentMetricKey, 2, SystemGroupKey) openFilePercentMetric := newMetricItem(OpenFilePercentMetricKey, 2, SystemGroupKey)
openFilePercentMetric.AddAxi("Open File Percent","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true) openFilePercentMetric.AddAxi("Open File Percent", "group1", common.PositionLeft, "ratio", "0.[0]", "0.[0]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "open_file_percent", Key: "open_file_percent",
Field: "payload.elasticsearch.node_stats.process.open_file_descriptors", Field: "payload.elasticsearch.node_stats.process.open_file_descriptors",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -293,7 +292,7 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
if value < 0 { if value < 0 {
return value return value
} }
return util.ToFixed((value / value2)*100, 2) return util.ToFixed((value/value2)*100, 2)
}, },
MetricItem: openFilePercentMetric, MetricItem: openFilePercentMetric,
FormatType: "ratio", FormatType: "ratio",
@ -301,9 +300,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case TotalDiskMetricKey: case TotalDiskMetricKey:
diskMetric := newMetricItem(TotalDiskMetricKey, 2, SystemGroupKey) diskMetric := newMetricItem(TotalDiskMetricKey, 2, SystemGroupKey)
diskMetric.AddAxi("disk available percent","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true) diskMetric.AddAxi("disk available percent", "group1", common.PositionLeft, "ratio", "0.[0]", "0.[0]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "disk", Key: "disk",
Field: "payload.elasticsearch.node_stats.fs.total.total_in_bytes", Field: "payload.elasticsearch.node_stats.fs.total.total_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -313,14 +312,14 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
Units: "%", Units: "%",
Field2: "payload.elasticsearch.node_stats.fs.total.available_in_bytes", Field2: "payload.elasticsearch.node_stats.fs.total.available_in_bytes",
Calc: func(value, value2 float64) float64 { Calc: func(value, value2 float64) float64 {
return util.ToFixed((value2 / value)*100, 2) return util.ToFixed((value2/value)*100, 2)
}, },
}) })
case IndexingRateMetricKey: case IndexingRateMetricKey:
// 索引速率 // 索引速率
indexMetric:=newMetricItem(IndexingRateMetricKey, 1, OperationGroupKey) indexMetric := newMetricItem(IndexingRateMetricKey, 1, OperationGroupKey)
indexMetric.AddAxi("indexing rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) indexMetric.AddAxi("indexing rate", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "indexing_rate", Key: "indexing_rate",
Field: "payload.elasticsearch.node_stats.indices.indexing.index_total", Field: "payload.elasticsearch.node_stats.indices.indexing.index_total",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -331,7 +330,7 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case IndexingBytesMetricKey: case IndexingBytesMetricKey:
indexingBytesMetric := newMetricItem(IndexingBytesMetricKey, 2, OperationGroupKey) indexingBytesMetric := newMetricItem(IndexingBytesMetricKey, 2, OperationGroupKey)
indexingBytesMetric.AddAxi("Indexing bytes","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) indexingBytesMetric.AddAxi("Indexing bytes", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
nodeMetricItems = append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "indexing_bytes", Key: "indexing_bytes",
Field: "payload.elasticsearch.node_stats.indices.store.size_in_bytes", Field: "payload.elasticsearch.node_stats.indices.store.size_in_bytes",
@ -343,14 +342,14 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case IndexingLatencyMetricKey: case IndexingLatencyMetricKey:
// 索引延时 // 索引延时
indexLatencyMetric:=newMetricItem(IndexingLatencyMetricKey, 1, LatencyGroupKey) indexLatencyMetric := newMetricItem(IndexingLatencyMetricKey, 1, LatencyGroupKey)
indexLatencyMetric.AddAxi("indexing latency","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) indexLatencyMetric.AddAxi("indexing latency", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "indexing_latency", Key: "indexing_latency",
Field: "payload.elasticsearch.node_stats.indices.indexing.index_time_in_millis", Field: "payload.elasticsearch.node_stats.indices.indexing.index_time_in_millis",
Field2: "payload.elasticsearch.node_stats.indices.indexing.index_total", Field2: "payload.elasticsearch.node_stats.indices.indexing.index_total",
Calc: func(value, value2 float64) float64 { Calc: func(value, value2 float64) float64 {
return value/value2 return value / value2
}, },
ID: util.GetUUID(), ID: util.GetUUID(),
IsDerivative: true, IsDerivative: true,
@ -359,9 +358,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
Units: "ms", Units: "ms",
}) })
case QueryRateMetricKey: case QueryRateMetricKey:
queryMetric:=newMetricItem(QueryRateMetricKey, 2, OperationGroupKey) queryMetric := newMetricItem(QueryRateMetricKey, 2, OperationGroupKey)
queryMetric.AddAxi("query rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) queryMetric.AddAxi("query rate", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "query_rate", Key: "query_rate",
Field: "payload.elasticsearch.node_stats.indices.search.query_total", Field: "payload.elasticsearch.node_stats.indices.search.query_total",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -372,14 +371,14 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case QueryLatencyMetricKey: case QueryLatencyMetricKey:
// 查询延时 // 查询延时
queryLatencyMetric:=newMetricItem(QueryLatencyMetricKey, 2, LatencyGroupKey) queryLatencyMetric := newMetricItem(QueryLatencyMetricKey, 2, LatencyGroupKey)
queryLatencyMetric.AddAxi("query latency","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) queryLatencyMetric.AddAxi("query latency", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "query_latency", Key: "query_latency",
Field: "payload.elasticsearch.node_stats.indices.search.query_time_in_millis", Field: "payload.elasticsearch.node_stats.indices.search.query_time_in_millis",
Field2: "payload.elasticsearch.node_stats.indices.search.query_total", Field2: "payload.elasticsearch.node_stats.indices.search.query_total",
Calc: func(value, value2 float64) float64 { Calc: func(value, value2 float64) float64 {
return value/value2 return value / value2
}, },
ID: util.GetUUID(), ID: util.GetUUID(),
IsDerivative: true, IsDerivative: true,
@ -388,9 +387,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
Units: "ms", Units: "ms",
}) })
case FetchRateMetricKey: case FetchRateMetricKey:
fetchMetric:=newMetricItem(FetchRateMetricKey, 3, OperationGroupKey) fetchMetric := newMetricItem(FetchRateMetricKey, 3, OperationGroupKey)
fetchMetric.AddAxi("fetch rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) fetchMetric.AddAxi("fetch rate", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "fetch_rate", Key: "fetch_rate",
Field: "payload.elasticsearch.node_stats.indices.search.fetch_total", Field: "payload.elasticsearch.node_stats.indices.search.fetch_total",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -400,9 +399,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
Units: "requests/s", Units: "requests/s",
}) })
case ScrollRateMetricKey: case ScrollRateMetricKey:
scrollMetric:=newMetricItem(ScrollRateMetricKey, 4, OperationGroupKey) scrollMetric := newMetricItem(ScrollRateMetricKey, 4, OperationGroupKey)
scrollMetric.AddAxi("scroll rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) scrollMetric.AddAxi("scroll rate", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "scroll_rate", Key: "scroll_rate",
Field: "payload.elasticsearch.node_stats.indices.search.scroll_total", Field: "payload.elasticsearch.node_stats.indices.search.scroll_total",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -412,9 +411,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
Units: "requests/s", Units: "requests/s",
}) })
case RefreshRateMetricKey: case RefreshRateMetricKey:
refreshMetric:=newMetricItem(RefreshRateMetricKey, 5, OperationGroupKey) refreshMetric := newMetricItem(RefreshRateMetricKey, 5, OperationGroupKey)
refreshMetric.AddAxi("refresh rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) refreshMetric.AddAxi("refresh rate", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "refresh_rate", Key: "refresh_rate",
Field: "payload.elasticsearch.node_stats.indices.refresh.total", Field: "payload.elasticsearch.node_stats.indices.refresh.total",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -424,9 +423,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
Units: "requests/s", Units: "requests/s",
}) })
case FlushRateMetricKey: case FlushRateMetricKey:
flushMetric:=newMetricItem(FlushRateMetricKey, 6, OperationGroupKey) flushMetric := newMetricItem(FlushRateMetricKey, 6, OperationGroupKey)
flushMetric.AddAxi("flush rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) flushMetric.AddAxi("flush rate", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "flush_rate", Key: "flush_rate",
Field: "payload.elasticsearch.node_stats.indices.flush.total", Field: "payload.elasticsearch.node_stats.indices.flush.total",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -436,9 +435,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
Units: "requests/s", Units: "requests/s",
}) })
case MergesRateMetricKey: case MergesRateMetricKey:
mergeMetric:=newMetricItem(MergesRateMetricKey, 7, OperationGroupKey) mergeMetric := newMetricItem(MergesRateMetricKey, 7, OperationGroupKey)
mergeMetric.AddAxi("merges rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) mergeMetric.AddAxi("merges rate", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "merges_rate", Key: "merges_rate",
Field: "payload.elasticsearch.node_stats.indices.merges.total", Field: "payload.elasticsearch.node_stats.indices.merges.total",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -449,14 +448,14 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case FetchLatencyMetricKey: case FetchLatencyMetricKey:
// fetch延时 // fetch延时
fetchLatencyMetric:=newMetricItem(FetchLatencyMetricKey, 3, LatencyGroupKey) fetchLatencyMetric := newMetricItem(FetchLatencyMetricKey, 3, LatencyGroupKey)
fetchLatencyMetric.AddAxi("fetch latency","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) fetchLatencyMetric.AddAxi("fetch latency", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "fetch_latency", Key: "fetch_latency",
Field: "payload.elasticsearch.node_stats.indices.search.fetch_time_in_millis", Field: "payload.elasticsearch.node_stats.indices.search.fetch_time_in_millis",
Field2: "payload.elasticsearch.node_stats.indices.search.fetch_total", Field2: "payload.elasticsearch.node_stats.indices.search.fetch_total",
Calc: func(value, value2 float64) float64 { Calc: func(value, value2 float64) float64 {
return value/value2 return value / value2
}, },
ID: util.GetUUID(), ID: util.GetUUID(),
IsDerivative: true, IsDerivative: true,
@ -466,14 +465,14 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case ScrollLatencyMetricKey: case ScrollLatencyMetricKey:
// scroll 延时 // scroll 延时
scrollLatencyMetric:=newMetricItem(ScrollLatencyMetricKey, 4, LatencyGroupKey) scrollLatencyMetric := newMetricItem(ScrollLatencyMetricKey, 4, LatencyGroupKey)
scrollLatencyMetric.AddAxi("scroll latency","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) scrollLatencyMetric.AddAxi("scroll latency", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "scroll_latency", Key: "scroll_latency",
Field: "payload.elasticsearch.node_stats.indices.search.scroll_time_in_millis", Field: "payload.elasticsearch.node_stats.indices.search.scroll_time_in_millis",
Field2: "payload.elasticsearch.node_stats.indices.search.scroll_total", Field2: "payload.elasticsearch.node_stats.indices.search.scroll_total",
Calc: func(value, value2 float64) float64 { Calc: func(value, value2 float64) float64 {
return value/value2 return value / value2
}, },
ID: util.GetUUID(), ID: util.GetUUID(),
IsDerivative: true, IsDerivative: true,
@ -483,14 +482,14 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case MergeLatencyMetricKey: case MergeLatencyMetricKey:
// merge 延时 // merge 延时
mergeLatencyMetric:=newMetricItem(MergeLatencyMetricKey, 7, LatencyGroupKey) mergeLatencyMetric := newMetricItem(MergeLatencyMetricKey, 7, LatencyGroupKey)
mergeLatencyMetric.AddAxi("merge latency","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) mergeLatencyMetric.AddAxi("merge latency", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "merge_latency", Key: "merge_latency",
Field: "payload.elasticsearch.node_stats.indices.merges.total_time_in_millis", Field: "payload.elasticsearch.node_stats.indices.merges.total_time_in_millis",
Field2: "payload.elasticsearch.node_stats.indices.merges.total", Field2: "payload.elasticsearch.node_stats.indices.merges.total",
Calc: func(value, value2 float64) float64 { Calc: func(value, value2 float64) float64 {
return value/value2 return value / value2
}, },
ID: util.GetUUID(), ID: util.GetUUID(),
IsDerivative: true, IsDerivative: true,
@ -500,14 +499,14 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case RefreshLatencyMetricKey: case RefreshLatencyMetricKey:
// refresh 延时 // refresh 延时
refreshLatencyMetric:=newMetricItem(RefreshLatencyMetricKey, 5, LatencyGroupKey) refreshLatencyMetric := newMetricItem(RefreshLatencyMetricKey, 5, LatencyGroupKey)
refreshLatencyMetric.AddAxi("refresh latency","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) refreshLatencyMetric.AddAxi("refresh latency", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "refresh_latency", Key: "refresh_latency",
Field: "payload.elasticsearch.node_stats.indices.refresh.total_time_in_millis", Field: "payload.elasticsearch.node_stats.indices.refresh.total_time_in_millis",
Field2: "payload.elasticsearch.node_stats.indices.refresh.total", Field2: "payload.elasticsearch.node_stats.indices.refresh.total",
Calc: func(value, value2 float64) float64 { Calc: func(value, value2 float64) float64 {
return value/value2 return value / value2
}, },
ID: util.GetUUID(), ID: util.GetUUID(),
IsDerivative: true, IsDerivative: true,
@ -517,14 +516,14 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case FlushLatencyMetricKey: case FlushLatencyMetricKey:
// flush 时延 // flush 时延
flushLatencyMetric:=newMetricItem(FlushLatencyMetricKey, 6, LatencyGroupKey) flushLatencyMetric := newMetricItem(FlushLatencyMetricKey, 6, LatencyGroupKey)
flushLatencyMetric.AddAxi("flush latency","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) flushLatencyMetric.AddAxi("flush latency", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "flush_latency", Key: "flush_latency",
Field: "payload.elasticsearch.node_stats.indices.flush.total_time_in_millis", Field: "payload.elasticsearch.node_stats.indices.flush.total_time_in_millis",
Field2: "payload.elasticsearch.node_stats.indices.flush.total", Field2: "payload.elasticsearch.node_stats.indices.flush.total",
Calc: func(value, value2 float64) float64 { Calc: func(value, value2 float64) float64 {
return value/value2 return value / value2
}, },
ID: util.GetUUID(), ID: util.GetUUID(),
IsDerivative: true, IsDerivative: true,
@ -534,9 +533,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case QueryCacheMetricKey: case QueryCacheMetricKey:
// Query Cache 内存占用大小 // Query Cache 内存占用大小
queryCacheMetric:=newMetricItem(QueryCacheMetricKey, 1, CacheGroupKey) queryCacheMetric := newMetricItem(QueryCacheMetricKey, 1, CacheGroupKey)
queryCacheMetric.AddAxi("query cache","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) queryCacheMetric.AddAxi("query cache", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "query_cache", Key: "query_cache",
Field: "payload.elasticsearch.node_stats.indices.query_cache.memory_size_in_bytes", Field: "payload.elasticsearch.node_stats.indices.query_cache.memory_size_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -547,9 +546,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case RequestCacheMetricKey: case RequestCacheMetricKey:
// Request Cache 内存占用大小 // Request Cache 内存占用大小
requestCacheMetric:=newMetricItem(RequestCacheMetricKey, 2, CacheGroupKey) requestCacheMetric := newMetricItem(RequestCacheMetricKey, 2, CacheGroupKey)
requestCacheMetric.AddAxi("request cache","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) requestCacheMetric.AddAxi("request cache", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "request_cache", Key: "request_cache",
Field: "payload.elasticsearch.node_stats.indices.request_cache.memory_size_in_bytes", Field: "payload.elasticsearch.node_stats.indices.request_cache.memory_size_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -560,9 +559,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case RequestCacheHitMetricKey: case RequestCacheHitMetricKey:
// Request Cache Hit // Request Cache Hit
requestCacheHitMetric:=newMetricItem(RequestCacheHitMetricKey, 6, CacheGroupKey) requestCacheHitMetric := newMetricItem(RequestCacheHitMetricKey, 6, CacheGroupKey)
requestCacheHitMetric.AddAxi("request cache hit","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) requestCacheHitMetric.AddAxi("request cache hit", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "request_cache_hit", Key: "request_cache_hit",
Field: "payload.elasticsearch.node_stats.indices.request_cache.hit_count", Field: "payload.elasticsearch.node_stats.indices.request_cache.hit_count",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -573,9 +572,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case RequestCacheMissMetricKey: case RequestCacheMissMetricKey:
// Request Cache Miss // Request Cache Miss
requestCacheMissMetric:=newMetricItem(RequestCacheMissMetricKey, 8, CacheGroupKey) requestCacheMissMetric := newMetricItem(RequestCacheMissMetricKey, 8, CacheGroupKey)
requestCacheMissMetric.AddAxi("request cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) requestCacheMissMetric.AddAxi("request cache miss", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "request_cache_miss", Key: "request_cache_miss",
Field: "payload.elasticsearch.node_stats.indices.request_cache.miss_count", Field: "payload.elasticsearch.node_stats.indices.request_cache.miss_count",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -586,9 +585,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case QueryCacheCountMetricKey: case QueryCacheCountMetricKey:
// Query Cache Count // Query Cache Count
queryCacheCountMetric:=newMetricItem(QueryCacheCountMetricKey, 4, CacheGroupKey) queryCacheCountMetric := newMetricItem(QueryCacheCountMetricKey, 4, CacheGroupKey)
queryCacheCountMetric.AddAxi("query cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) queryCacheCountMetric.AddAxi("query cache miss", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "query_cache_count", Key: "query_cache_count",
Field: "payload.elasticsearch.node_stats.indices.query_cache.cache_count", Field: "payload.elasticsearch.node_stats.indices.query_cache.cache_count",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -598,9 +597,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
Units: "", Units: "",
}) })
case QueryCacheHitMetricKey: case QueryCacheHitMetricKey:
queryCacheHitMetric:=newMetricItem(QueryCacheHitMetricKey, 5, CacheGroupKey) queryCacheHitMetric := newMetricItem(QueryCacheHitMetricKey, 5, CacheGroupKey)
queryCacheHitMetric.AddAxi("query cache hit","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) queryCacheHitMetric.AddAxi("query cache hit", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "query_cache_hit", Key: "query_cache_hit",
Field: "payload.elasticsearch.node_stats.indices.query_cache.hit_count", Field: "payload.elasticsearch.node_stats.indices.query_cache.hit_count",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -611,9 +610,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case QueryCacheMissMetricKey: case QueryCacheMissMetricKey:
// Query Cache Miss // Query Cache Miss
queryCacheMissMetric:=newMetricItem(QueryCacheMissMetricKey, 7, CacheGroupKey) queryCacheMissMetric := newMetricItem(QueryCacheMissMetricKey, 7, CacheGroupKey)
queryCacheMissMetric.AddAxi("query cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) queryCacheMissMetric.AddAxi("query cache miss", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "query_cache_miss", Key: "query_cache_miss",
Field: "payload.elasticsearch.node_stats.indices.query_cache.miss_count", Field: "payload.elasticsearch.node_stats.indices.query_cache.miss_count",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -624,9 +623,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case FielddataCacheMetricKey: case FielddataCacheMetricKey:
// Fielddata内存占用大小 // Fielddata内存占用大小
fieldDataCacheMetric:=newMetricItem(FielddataCacheMetricKey, 3, CacheGroupKey) fieldDataCacheMetric := newMetricItem(FielddataCacheMetricKey, 3, CacheGroupKey)
fieldDataCacheMetric.AddAxi("FieldData Cache","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) fieldDataCacheMetric.AddAxi("FieldData Cache", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "fielddata_cache", Key: "fielddata_cache",
Field: "payload.elasticsearch.node_stats.indices.fielddata.memory_size_in_bytes", Field: "payload.elasticsearch.node_stats.indices.fielddata.memory_size_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -637,9 +636,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case HttpConnectNumMetricKey: case HttpConnectNumMetricKey:
// http 活跃连接数 // http 活跃连接数
httpActiveMetric:=newMetricItem(HttpConnectNumMetricKey, 12, HttpGroupKey) httpActiveMetric := newMetricItem(HttpConnectNumMetricKey, 12, HttpGroupKey)
httpActiveMetric.AddAxi("http connect number","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) httpActiveMetric.AddAxi("http connect number", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "http_connect_num", Key: "http_connect_num",
Field: "payload.elasticsearch.node_stats.http.current_open", Field: "payload.elasticsearch.node_stats.http.current_open",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -650,9 +649,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case HttpRateMetricKey: case HttpRateMetricKey:
// http 活跃连接数速率 // http 活跃连接数速率
httpRateMetric:=newMetricItem(HttpRateMetricKey, 12, HttpGroupKey) httpRateMetric := newMetricItem(HttpRateMetricKey, 12, HttpGroupKey)
httpRateMetric.AddAxi("http rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) httpRateMetric.AddAxi("http rate", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "http_rate", Key: "http_rate",
Field: "payload.elasticsearch.node_stats.http.total_opened", Field: "payload.elasticsearch.node_stats.http.total_opened",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -663,9 +662,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case SegmentCountMetricKey: case SegmentCountMetricKey:
// segment 数量 // segment 数量
segmentCountMetric:=newMetricItem(SegmentCountMetricKey, 15, StorageGroupKey) segmentCountMetric := newMetricItem(SegmentCountMetricKey, 15, StorageGroupKey)
segmentCountMetric.AddAxi("segment count","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) segmentCountMetric.AddAxi("segment count", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "segment_count", Key: "segment_count",
Field: "payload.elasticsearch.node_stats.indices.segments.count", Field: "payload.elasticsearch.node_stats.indices.segments.count",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -676,9 +675,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case SegmentMemoryMetricKey: case SegmentMemoryMetricKey:
// segment memory // segment memory
segmentMemoryMetric:=newMetricItem(SegmentMemoryMetricKey, 16, MemoryGroupKey) segmentMemoryMetric := newMetricItem(SegmentMemoryMetricKey, 16, MemoryGroupKey)
segmentMemoryMetric.AddAxi("segment memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) segmentMemoryMetric.AddAxi("segment memory", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "segment_memory", Key: "segment_memory",
Field: "payload.elasticsearch.node_stats.indices.segments.memory_in_bytes", Field: "payload.elasticsearch.node_stats.indices.segments.memory_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -689,9 +688,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case SegmentStoredFieldsMemoryMetricKey: case SegmentStoredFieldsMemoryMetricKey:
// segment stored fields memory // segment stored fields memory
segmentStoredFieldsMemoryMetric:=newMetricItem(SegmentStoredFieldsMemoryMetricKey, 16, MemoryGroupKey) segmentStoredFieldsMemoryMetric := newMetricItem(SegmentStoredFieldsMemoryMetricKey, 16, MemoryGroupKey)
segmentStoredFieldsMemoryMetric.AddAxi("segment stored fields memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) segmentStoredFieldsMemoryMetric.AddAxi("segment stored fields memory", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "segment_stored_fields_memory", Key: "segment_stored_fields_memory",
Field: "payload.elasticsearch.node_stats.indices.segments.stored_fields_memory_in_bytes", Field: "payload.elasticsearch.node_stats.indices.segments.stored_fields_memory_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -702,9 +701,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case SegmentTermsMemoryMetricKey: case SegmentTermsMemoryMetricKey:
// segment terms fields memory // segment terms fields memory
segmentTermsMemoryMetric:=newMetricItem(SegmentTermsMemoryMetricKey, 16, MemoryGroupKey) segmentTermsMemoryMetric := newMetricItem(SegmentTermsMemoryMetricKey, 16, MemoryGroupKey)
segmentTermsMemoryMetric.AddAxi("segment terms memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) segmentTermsMemoryMetric.AddAxi("segment terms memory", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "segment_terms_memory", Key: "segment_terms_memory",
Field: "payload.elasticsearch.node_stats.indices.segments.terms_memory_in_bytes", Field: "payload.elasticsearch.node_stats.indices.segments.terms_memory_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -715,9 +714,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case SegmentDocValuesMemoryMetricKey: case SegmentDocValuesMemoryMetricKey:
// segment doc values memory // segment doc values memory
segmentDocValuesMemoryMetric:=newMetricItem(SegmentDocValuesMemoryMetricKey, 16, MemoryGroupKey) segmentDocValuesMemoryMetric := newMetricItem(SegmentDocValuesMemoryMetricKey, 16, MemoryGroupKey)
segmentDocValuesMemoryMetric.AddAxi("segment doc values memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) segmentDocValuesMemoryMetric.AddAxi("segment doc values memory", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "segment_doc_values_memory", Key: "segment_doc_values_memory",
Field: "payload.elasticsearch.node_stats.indices.segments.doc_values_memory_in_bytes", Field: "payload.elasticsearch.node_stats.indices.segments.doc_values_memory_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -728,9 +727,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case SegmentIndexWriterMemoryMetricKey: case SegmentIndexWriterMemoryMetricKey:
// segment index writer memory // segment index writer memory
segmentIndexWriterMemoryMetric:=newMetricItem(SegmentIndexWriterMemoryMetricKey, 16, MemoryGroupKey) segmentIndexWriterMemoryMetric := newMetricItem(SegmentIndexWriterMemoryMetricKey, 16, MemoryGroupKey)
segmentIndexWriterMemoryMetric.AddAxi("segment doc values memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) segmentIndexWriterMemoryMetric.AddAxi("segment doc values memory", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "segment_index_writer_memory", Key: "segment_index_writer_memory",
Field: "payload.elasticsearch.node_stats.indices.segments.index_writer_memory_in_bytes", Field: "payload.elasticsearch.node_stats.indices.segments.index_writer_memory_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -741,9 +740,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case SegmentTermVectorsMemoryMetricKey: case SegmentTermVectorsMemoryMetricKey:
// segment term vectors memory // segment term vectors memory
segmentTermVectorsMemoryMetric:=newMetricItem(SegmentTermVectorsMemoryMetricKey, 16, MemoryGroupKey) segmentTermVectorsMemoryMetric := newMetricItem(SegmentTermVectorsMemoryMetricKey, 16, MemoryGroupKey)
segmentTermVectorsMemoryMetric.AddAxi("segment term vectors memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) segmentTermVectorsMemoryMetric.AddAxi("segment term vectors memory", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "segment_term_vectors_memory", Key: "segment_term_vectors_memory",
Field: "payload.elasticsearch.node_stats.indices.segments.term_vectors_memory_in_bytes", Field: "payload.elasticsearch.node_stats.indices.segments.term_vectors_memory_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -754,9 +753,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case DocsCountMetricKey: case DocsCountMetricKey:
// docs 数量 // docs 数量
docsCountMetric:=newMetricItem(DocsCountMetricKey, 17, DocumentGroupKey) docsCountMetric := newMetricItem(DocsCountMetricKey, 17, DocumentGroupKey)
docsCountMetric.AddAxi("docs count","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) docsCountMetric.AddAxi("docs count", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "docs_count", Key: "docs_count",
Field: "payload.elasticsearch.node_stats.indices.docs.count", Field: "payload.elasticsearch.node_stats.indices.docs.count",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -767,9 +766,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case DocsDeletedMetricKey: case DocsDeletedMetricKey:
// docs 删除数量 // docs 删除数量
docsDeletedMetric:=newMetricItem(DocsDeletedMetricKey, 17, DocumentGroupKey) docsDeletedMetric := newMetricItem(DocsDeletedMetricKey, 17, DocumentGroupKey)
docsDeletedMetric.AddAxi("docs deleted","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) docsDeletedMetric.AddAxi("docs deleted", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "docs_deleted", Key: "docs_deleted",
Field: "payload.elasticsearch.node_stats.indices.docs.deleted", Field: "payload.elasticsearch.node_stats.indices.docs.deleted",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -780,9 +779,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case IndexStorageMetricKey: case IndexStorageMetricKey:
// index store size // index store size
indexStoreMetric:=newMetricItem(IndexStorageMetricKey, 18, StorageGroupKey) indexStoreMetric := newMetricItem(IndexStorageMetricKey, 18, StorageGroupKey)
indexStoreMetric.AddAxi("indices storage","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) indexStoreMetric.AddAxi("indices storage", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "index_storage", Key: "index_storage",
Field: "payload.elasticsearch.node_stats.indices.store.size_in_bytes", Field: "payload.elasticsearch.node_stats.indices.store.size_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -793,9 +792,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case JVMHeapUsedPercentMetricKey: case JVMHeapUsedPercentMetricKey:
// jvm used heap // jvm used heap
jvmUsedPercentMetric:=newMetricItem(JVMHeapUsedPercentMetricKey, 1, JVMGroupKey) jvmUsedPercentMetric := newMetricItem(JVMHeapUsedPercentMetricKey, 1, JVMGroupKey)
jvmUsedPercentMetric.AddAxi("JVM heap used percent","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) jvmUsedPercentMetric.AddAxi("JVM heap used percent", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "jvm_heap_used_percent", Key: "jvm_heap_used_percent",
Field: "payload.elasticsearch.node_stats.jvm.mem.heap_used_percent", Field: "payload.elasticsearch.node_stats.jvm.mem.heap_used_percent",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -806,9 +805,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case JVMMemYoungUsedMetricKey: case JVMMemYoungUsedMetricKey:
//JVM mem Young pools used //JVM mem Young pools used
youngPoolsUsedMetric:=newMetricItem(JVMMemYoungUsedMetricKey, 2, JVMGroupKey) youngPoolsUsedMetric := newMetricItem(JVMMemYoungUsedMetricKey, 2, JVMGroupKey)
youngPoolsUsedMetric.AddAxi("Mem Pools Young Used","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) youngPoolsUsedMetric.AddAxi("Mem Pools Young Used", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "jvm_mem_young_used", Key: "jvm_mem_young_used",
Field: "payload.elasticsearch.node_stats.jvm.mem.pools.young.used_in_bytes", Field: "payload.elasticsearch.node_stats.jvm.mem.pools.young.used_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -819,9 +818,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case JVMMemYoungPeakUsedMetricKey: case JVMMemYoungPeakUsedMetricKey:
//JVM mem Young pools peak used //JVM mem Young pools peak used
youngPoolsUsedPeakMetric:=newMetricItem(JVMMemYoungPeakUsedMetricKey, 2, JVMGroupKey) youngPoolsUsedPeakMetric := newMetricItem(JVMMemYoungPeakUsedMetricKey, 2, JVMGroupKey)
youngPoolsUsedPeakMetric.AddAxi("Mem Pools Young Peak Used","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) youngPoolsUsedPeakMetric.AddAxi("Mem Pools Young Peak Used", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "jvm_mem_young_peak_used", Key: "jvm_mem_young_peak_used",
Field: "payload.elasticsearch.node_stats.jvm.mem.pools.young.peak_used_in_bytes", Field: "payload.elasticsearch.node_stats.jvm.mem.pools.young.peak_used_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -832,9 +831,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case JVMMemOldUsedMetricKey: case JVMMemOldUsedMetricKey:
//JVM mem old pools used //JVM mem old pools used
oldPoolsUsedMetric:=newMetricItem(JVMMemOldUsedMetricKey, 3, JVMGroupKey) oldPoolsUsedMetric := newMetricItem(JVMMemOldUsedMetricKey, 3, JVMGroupKey)
oldPoolsUsedMetric.AddAxi("Mem Pools Old Used","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) oldPoolsUsedMetric.AddAxi("Mem Pools Old Used", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "jvm_mem_old_used", Key: "jvm_mem_old_used",
Field: "payload.elasticsearch.node_stats.jvm.mem.pools.old.used_in_bytes", Field: "payload.elasticsearch.node_stats.jvm.mem.pools.old.used_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -845,9 +844,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case JVMMemOldPeakUsedMetricKey: case JVMMemOldPeakUsedMetricKey:
//JVM mem old pools peak used //JVM mem old pools peak used
oldPoolsUsedPeakMetric:=newMetricItem(JVMMemOldPeakUsedMetricKey, 3, JVMGroupKey) oldPoolsUsedPeakMetric := newMetricItem(JVMMemOldPeakUsedMetricKey, 3, JVMGroupKey)
oldPoolsUsedPeakMetric.AddAxi("Mem Pools Old Peak Used","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) oldPoolsUsedPeakMetric.AddAxi("Mem Pools Old Peak Used", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "jvm_mem_old_peak_used", Key: "jvm_mem_old_peak_used",
Field: "payload.elasticsearch.node_stats.jvm.mem.pools.old.peak_used_in_bytes", Field: "payload.elasticsearch.node_stats.jvm.mem.pools.old.peak_used_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -858,9 +857,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case JVMUsedHeapMetricKey: case JVMUsedHeapMetricKey:
//JVM used heap //JVM used heap
heapUsedMetric:=newMetricItem(JVMUsedHeapMetricKey, 1, JVMGroupKey) heapUsedMetric := newMetricItem(JVMUsedHeapMetricKey, 1, JVMGroupKey)
heapUsedMetric.AddAxi("JVM Used Heap","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) heapUsedMetric.AddAxi("JVM Used Heap", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "jvm_used_heap", Key: "jvm_used_heap",
Field: "payload.elasticsearch.node_stats.jvm.mem.heap_used_in_bytes", Field: "payload.elasticsearch.node_stats.jvm.mem.heap_used_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -871,9 +870,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case JVMYoungGCRateMetricKey: case JVMYoungGCRateMetricKey:
//JVM Young GC Rate //JVM Young GC Rate
gcYoungRateMetric:=newMetricItem(JVMYoungGCRateMetricKey, 2, JVMGroupKey) gcYoungRateMetric := newMetricItem(JVMYoungGCRateMetricKey, 2, JVMGroupKey)
gcYoungRateMetric.AddAxi("JVM Young GC Rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) gcYoungRateMetric.AddAxi("JVM Young GC Rate", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "jvm_young_gc_rate", Key: "jvm_young_gc_rate",
Field: "payload.elasticsearch.node_stats.jvm.gc.collectors.young.collection_count", Field: "payload.elasticsearch.node_stats.jvm.gc.collectors.young.collection_count",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -884,9 +883,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case JVMYoungGCLatencyMetricKey: case JVMYoungGCLatencyMetricKey:
//JVM Young GC Latency //JVM Young GC Latency
gcYoungLatencyMetric:=newMetricItem(JVMYoungGCLatencyMetricKey, 2, JVMGroupKey) gcYoungLatencyMetric := newMetricItem(JVMYoungGCLatencyMetricKey, 2, JVMGroupKey)
gcYoungLatencyMetric.AddAxi("JVM Young GC Time","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) gcYoungLatencyMetric.AddAxi("JVM Young GC Time", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "jvm_young_gc_latency", Key: "jvm_young_gc_latency",
Field: "payload.elasticsearch.node_stats.jvm.gc.collectors.young.collection_time_in_millis", Field: "payload.elasticsearch.node_stats.jvm.gc.collectors.young.collection_time_in_millis",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -897,9 +896,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case JVMOldGCRateMetricKey: case JVMOldGCRateMetricKey:
//JVM old GC Rate //JVM old GC Rate
gcOldRateMetric:=newMetricItem(JVMOldGCRateMetricKey, 3, JVMGroupKey) gcOldRateMetric := newMetricItem(JVMOldGCRateMetricKey, 3, JVMGroupKey)
gcOldRateMetric.AddAxi("JVM Old GC Rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) gcOldRateMetric.AddAxi("JVM Old GC Rate", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "jvm_old_gc_rate", Key: "jvm_old_gc_rate",
Field: "payload.elasticsearch.node_stats.jvm.gc.collectors.old.collection_count", Field: "payload.elasticsearch.node_stats.jvm.gc.collectors.old.collection_count",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -910,9 +909,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case JVMOldGCLatencyMetricKey: case JVMOldGCLatencyMetricKey:
//JVM old GC Latency //JVM old GC Latency
gcOldLatencyMetric:=newMetricItem(JVMOldGCLatencyMetricKey, 3, JVMGroupKey) gcOldLatencyMetric := newMetricItem(JVMOldGCLatencyMetricKey, 3, JVMGroupKey)
gcOldLatencyMetric.AddAxi("JVM Old GC Time","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) gcOldLatencyMetric.AddAxi("JVM Old GC Time", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "jvm_old_gc_latency", Key: "jvm_old_gc_latency",
Field: "payload.elasticsearch.node_stats.jvm.gc.collectors.old.collection_time_in_millis", Field: "payload.elasticsearch.node_stats.jvm.gc.collectors.old.collection_time_in_millis",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -923,9 +922,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case TransportTXRateMetricKey: case TransportTXRateMetricKey:
//Transport 发送速率 //Transport 发送速率
transTxRateMetric:=newMetricItem(TransportTXRateMetricKey, 19, TransportGroupKey) transTxRateMetric := newMetricItem(TransportTXRateMetricKey, 19, TransportGroupKey)
transTxRateMetric.AddAxi("Transport Send Rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) transTxRateMetric.AddAxi("Transport Send Rate", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "transport_tx_rate", Key: "transport_tx_rate",
Field: "payload.elasticsearch.node_stats.transport.tx_count", Field: "payload.elasticsearch.node_stats.transport.tx_count",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -936,9 +935,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case TransportRXRateMetricKey: case TransportRXRateMetricKey:
//Transport 接收速率 //Transport 接收速率
transRxRateMetric:=newMetricItem(TransportRXRateMetricKey, 19, TransportGroupKey) transRxRateMetric := newMetricItem(TransportRXRateMetricKey, 19, TransportGroupKey)
transRxRateMetric.AddAxi("Transport Receive Rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) transRxRateMetric.AddAxi("Transport Receive Rate", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "transport_rx_rate", Key: "transport_rx_rate",
Field: "payload.elasticsearch.node_stats.transport.rx_count", Field: "payload.elasticsearch.node_stats.transport.rx_count",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -949,9 +948,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case TransportTXBytesMetricKey: case TransportTXBytesMetricKey:
//Transport 发送流量 //Transport 发送流量
transTxBytesMetric:=newMetricItem(TransportTXBytesMetricKey, 19, TransportGroupKey) transTxBytesMetric := newMetricItem(TransportTXBytesMetricKey, 19, TransportGroupKey)
transTxBytesMetric.AddAxi("Transport Send Bytes","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) transTxBytesMetric.AddAxi("Transport Send Bytes", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "transport_tx_bytes", Key: "transport_tx_bytes",
Field: "payload.elasticsearch.node_stats.transport.tx_size_in_bytes", Field: "payload.elasticsearch.node_stats.transport.tx_size_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -962,9 +961,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case TransportRXBytesMetricKey: case TransportRXBytesMetricKey:
//Transport 接收流量 //Transport 接收流量
transRxBytesMetric:=newMetricItem(TransportRXBytesMetricKey, 19, TransportGroupKey) transRxBytesMetric := newMetricItem(TransportRXBytesMetricKey, 19, TransportGroupKey)
transRxBytesMetric.AddAxi("Transport Receive Bytes","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) transRxBytesMetric.AddAxi("Transport Receive Bytes", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "transport_rx_bytes", Key: "transport_rx_bytes",
Field: "payload.elasticsearch.node_stats.transport.rx_size_in_bytes", Field: "payload.elasticsearch.node_stats.transport.rx_size_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -975,9 +974,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case TransportTCPOutboundMetricKey: case TransportTCPOutboundMetricKey:
//Transport tcp 连接数 //Transport tcp 连接数
tcpNumMetric:=newMetricItem(TransportTCPOutboundMetricKey, 20, TransportGroupKey) tcpNumMetric := newMetricItem(TransportTCPOutboundMetricKey, 20, TransportGroupKey)
tcpNumMetric.AddAxi("Transport Outbound Connections","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) tcpNumMetric.AddAxi("Transport Outbound Connections", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "transport_outbound_connections", Key: "transport_outbound_connections",
Field: "payload.elasticsearch.node_stats.transport.total_outbound_connections", Field: "payload.elasticsearch.node_stats.transport.total_outbound_connections",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -988,9 +987,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case TotalIOOperationsMetricKey: case TotalIOOperationsMetricKey:
//IO total //IO total
totalOperationsMetric:=newMetricItem(TotalIOOperationsMetricKey, 1, IOGroupKey) totalOperationsMetric := newMetricItem(TotalIOOperationsMetricKey, 1, IOGroupKey)
totalOperationsMetric.AddAxi("Total I/O Operations Rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) totalOperationsMetric.AddAxi("Total I/O Operations Rate", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "total_io_operations", Key: "total_io_operations",
Field: "payload.elasticsearch.node_stats.fs.io_stats.total.operations", Field: "payload.elasticsearch.node_stats.fs.io_stats.total.operations",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -1000,9 +999,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
Units: "times/s", Units: "times/s",
}) })
case TotalReadIOOperationsMetricKey: case TotalReadIOOperationsMetricKey:
readOperationsMetric:=newMetricItem(TotalReadIOOperationsMetricKey, 2, IOGroupKey) readOperationsMetric := newMetricItem(TotalReadIOOperationsMetricKey, 2, IOGroupKey)
readOperationsMetric.AddAxi("Total Read I/O Operations Rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) readOperationsMetric.AddAxi("Total Read I/O Operations Rate", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "total_read_io_operations", Key: "total_read_io_operations",
Field: "payload.elasticsearch.node_stats.fs.io_stats.total.read_operations", Field: "payload.elasticsearch.node_stats.fs.io_stats.total.read_operations",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -1012,9 +1011,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
Units: "times/s", Units: "times/s",
}) })
case TotalWriteIOOperationsMetricKey: case TotalWriteIOOperationsMetricKey:
writeOperationsMetric:=newMetricItem(TotalWriteIOOperationsMetricKey, 3, IOGroupKey) writeOperationsMetric := newMetricItem(TotalWriteIOOperationsMetricKey, 3, IOGroupKey)
writeOperationsMetric.AddAxi("Total Write I/O Operations Rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) writeOperationsMetric.AddAxi("Total Write I/O Operations Rate", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "total_write_io_operations", Key: "total_write_io_operations",
Field: "payload.elasticsearch.node_stats.fs.io_stats.total.write_operations", Field: "payload.elasticsearch.node_stats.fs.io_stats.total.write_operations",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -1025,9 +1024,9 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case ScrollOpenContextsMetricKey: case ScrollOpenContextsMetricKey:
//scroll context //scroll context
openContextMetric:=newMetricItem(ScrollOpenContextsMetricKey, 7, OperationGroupKey) openContextMetric := newMetricItem(ScrollOpenContextsMetricKey, 7, OperationGroupKey)
openContextMetric.AddAxi("Scroll Open Contexts","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) openContextMetric.AddAxi("Scroll Open Contexts", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "scroll_open_contexts", Key: "scroll_open_contexts",
Field: "payload.elasticsearch.node_stats.indices.search.open_contexts", Field: "payload.elasticsearch.node_stats.indices.search.open_contexts",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -1038,7 +1037,7 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
case ParentBreakerMetricKey: case ParentBreakerMetricKey:
// Circuit Breaker // Circuit Breaker
parentBreakerMetric := newMetricItem(ParentBreakerMetricKey, 1, CircuitBreakerGroupKey) parentBreakerMetric := newMetricItem(ParentBreakerMetricKey, 1, CircuitBreakerGroupKey)
parentBreakerMetric.AddAxi("Parent Breaker","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) parentBreakerMetric.AddAxi("Parent Breaker", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
nodeMetricItems = append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "parent_breaker", Key: "parent_breaker",
Field: "payload.elasticsearch.node_stats.breakers.parent.tripped", Field: "payload.elasticsearch.node_stats.breakers.parent.tripped",
@ -1050,7 +1049,7 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case AccountingBreakerMetricKey: case AccountingBreakerMetricKey:
accountingBreakerMetric := newMetricItem(AccountingBreakerMetricKey, 2, CircuitBreakerGroupKey) accountingBreakerMetric := newMetricItem(AccountingBreakerMetricKey, 2, CircuitBreakerGroupKey)
accountingBreakerMetric.AddAxi("Accounting Breaker","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) accountingBreakerMetric.AddAxi("Accounting Breaker", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
nodeMetricItems = append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "accounting_breaker", Key: "accounting_breaker",
Field: "payload.elasticsearch.node_stats.breakers.accounting.tripped", Field: "payload.elasticsearch.node_stats.breakers.accounting.tripped",
@ -1062,7 +1061,7 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case FielddataBreakerMetricKey: case FielddataBreakerMetricKey:
fielddataBreakerMetric := newMetricItem(FielddataBreakerMetricKey, 3, CircuitBreakerGroupKey) fielddataBreakerMetric := newMetricItem(FielddataBreakerMetricKey, 3, CircuitBreakerGroupKey)
fielddataBreakerMetric.AddAxi("Fielddata Breaker","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) fielddataBreakerMetric.AddAxi("Fielddata Breaker", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
nodeMetricItems = append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "fielddata_breaker", Key: "fielddata_breaker",
Field: "payload.elasticsearch.node_stats.breakers.fielddata.tripped", Field: "payload.elasticsearch.node_stats.breakers.fielddata.tripped",
@ -1074,7 +1073,7 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case RequestBreakerMetricKey: case RequestBreakerMetricKey:
requestBreakerMetric := newMetricItem(RequestBreakerMetricKey, 4, CircuitBreakerGroupKey) requestBreakerMetric := newMetricItem(RequestBreakerMetricKey, 4, CircuitBreakerGroupKey)
requestBreakerMetric.AddAxi("Request Breaker","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) requestBreakerMetric.AddAxi("Request Breaker", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
nodeMetricItems = append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "request_breaker", Key: "request_breaker",
Field: "payload.elasticsearch.node_stats.breakers.request.tripped", Field: "payload.elasticsearch.node_stats.breakers.request.tripped",
@ -1086,7 +1085,7 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
}) })
case InFlightRequestsBreakerMetricKey: case InFlightRequestsBreakerMetricKey:
inFlightRequestBreakerMetric := newMetricItem(InFlightRequestsBreakerMetricKey, 5, CircuitBreakerGroupKey) inFlightRequestBreakerMetric := newMetricItem(InFlightRequestsBreakerMetricKey, 5, CircuitBreakerGroupKey)
inFlightRequestBreakerMetric.AddAxi("In Flight Requests Breaker","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) inFlightRequestBreakerMetric.AddAxi("In Flight Requests Breaker", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
nodeMetricItems = append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "in_flight_requests_breaker", Key: "in_flight_requests_breaker",
Field: "payload.elasticsearch.node_stats.breakers.in_flight_requests.tripped", Field: "payload.elasticsearch.node_stats.breakers.in_flight_requests.tripped",
@ -1099,7 +1098,7 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
case ModelInferenceBreakerMetricKey: case ModelInferenceBreakerMetricKey:
//Elasticsearch 8.6+ Model Inference Breaker //Elasticsearch 8.6+ Model Inference Breaker
modelInferenceBreakerMetric := newMetricItem(ModelInferenceBreakerMetricKey, 6, CircuitBreakerGroupKey) modelInferenceBreakerMetric := newMetricItem(ModelInferenceBreakerMetricKey, 6, CircuitBreakerGroupKey)
modelInferenceBreakerMetric.AddAxi("Model Inference Breaker","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) modelInferenceBreakerMetric.AddAxi("Model Inference Breaker", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
nodeMetricItems = append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "model_inference_breaker", Key: "model_inference_breaker",
Field: "payload.elasticsearch.node_stats.breakers.model_inference.tripped", Field: "payload.elasticsearch.node_stats.breakers.model_inference.tripped",
@ -1140,7 +1139,7 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke
} }
func (h *APIHandler) getTopNodeName(clusterID string, top int, lastMinutes int) ([]string, error){ func (h *APIHandler) getTopNodeName(clusterID string, top int, lastMinutes int) ([]string, error) {
ver := h.Client().GetVersion() ver := h.Client().GetVersion()
cr, _ := util.VersionCompare(ver.Number, "6.1") cr, _ := util.VersionCompare(ver.Number, "6.1")
if (ver.Distribution == "" || ver.Distribution == elastic.Elasticsearch) && cr == -1 { if (ver.Distribution == "" || ver.Distribution == elastic.Elasticsearch) && cr == -1 {
@ -1148,8 +1147,8 @@ func (h *APIHandler) getTopNodeName(clusterID string, top int, lastMinutes int)
} }
var ( var (
now = time.Now() now = time.Now()
max = now.UnixNano()/1e6 max = now.UnixNano() / 1e6
min = now.Add(-time.Duration(lastMinutes) * time.Minute).UnixNano()/1e6 min = now.Add(-time.Duration(lastMinutes)*time.Minute).UnixNano() / 1e6
bucketSizeStr = "60s" bucketSizeStr = "60s"
) )
intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr)
@ -1276,20 +1275,20 @@ func (h *APIHandler) getTopNodeName(clusterID string, top int, lastMinutes int)
}, },
}, },
} }
response,err:=elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(),util.MustToJSONBytes(query)) response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query))
if err!=nil{ if err != nil {
log.Error(err) log.Error(err)
return nil, err return nil, err
} }
var maxQpsKVS = map[string] float64{} var maxQpsKVS = map[string]float64{}
for _, agg := range response.Aggregations { for _, agg := range response.Aggregations {
for _, bk := range agg.Buckets { for _, bk := range agg.Buckets {
key := bk["key"].(string) key := bk["key"].(string)
if maxQps, ok := bk["max_qps"].(map[string]interface{}); ok { if maxQps, ok := bk["max_qps"].(map[string]interface{}); ok {
val := maxQps["value"].(float64) val := maxQps["value"].(float64)
if _, ok = maxQpsKVS[key] ; ok { if _, ok = maxQpsKVS[key]; ok {
maxQpsKVS[key] = maxQpsKVS[key] + val maxQpsKVS[key] = maxQpsKVS[key] + val
}else{ } else {
maxQpsKVS[key] = val maxQpsKVS[key] = val
} }
} }
@ -1310,7 +1309,7 @@ func (h *APIHandler) getTopNodeName(clusterID string, top int, lastMinutes int)
length = len(qpsValues) length = len(qpsValues)
} }
nodeNames := []string{} nodeNames := []string{}
for i := 0; i <length; i++ { for i := 0; i < length; i++ {
nodeNames = append(nodeNames, qpsValues[i].Key) nodeNames = append(nodeNames, qpsValues[i].Key)
} }
return nodeNames, nil return nodeNames, nil

222
modules/elastic/api/node_overview.go
View File

@ -45,8 +45,8 @@ import (
) )
func (h *APIHandler) SearchNodeMetadata(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *APIHandler) SearchNodeMetadata(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody:=util.MapStr{} resBody := util.MapStr{}
reqBody := struct{ reqBody := struct {
Keyword string `json:"keyword"` Keyword string `json:"keyword"`
Size int `json:"size"` Size int `json:"size"`
From int `json:"from"` From int `json:"from"`
@ -59,7 +59,7 @@ func (h *APIHandler) SearchNodeMetadata(w http.ResponseWriter, req *http.Request
err := h.DecodeJSON(req, &reqBody) err := h.DecodeJSON(req, &reqBody)
if err != nil { if err != nil {
resBody["error"] = err.Error() resBody["error"] = err.Error()
h.WriteJSON(w,resBody, http.StatusInternalServerError ) h.WriteJSON(w, resBody, http.StatusInternalServerError)
return return
} }
aggs := elastic.BuildSearchTermAggregations(reqBody.Aggregations) aggs := elastic.BuildSearchTermAggregations(reqBody.Aggregations)
@ -77,8 +77,8 @@ func (h *APIHandler) SearchNodeMetadata(w http.ResponseWriter, req *http.Request
}, },
}, },
} }
var should =[]util.MapStr{} var should = []util.MapStr{}
if reqBody.SearchField != ""{ if reqBody.SearchField != "" {
should = []util.MapStr{ should = []util.MapStr{
{ {
"prefix": util.MapStr{ "prefix": util.MapStr{
@ -101,7 +101,7 @@ func (h *APIHandler) SearchNodeMetadata(w http.ResponseWriter, req *http.Request
}, },
}, },
} }
}else{ } else {
should = []util.MapStr{ should = []util.MapStr{
{ {
"prefix": util.MapStr{ "prefix": util.MapStr{
@ -143,19 +143,14 @@ func (h *APIHandler) SearchNodeMetadata(w http.ResponseWriter, req *http.Request
} }
clusterFilter, hasPrivilege := h.GetClusterFilter(req, "metadata.cluster_id") clusterFilter, hasPrivilege := h.GetClusterFilter(req, "metadata.cluster_id")
if !hasPrivilege && clusterFilter == nil { if !hasPrivilege && clusterFilter == nil {
h.WriteJSON(w, elastic.SearchResponse{ h.WriteJSON(w, elastic.SearchResponse{}, http.StatusOK)
}, http.StatusOK)
return return
} }
must := []interface{}{ must := []interface{}{}
}
if !hasPrivilege && clusterFilter != nil { if !hasPrivilege && clusterFilter != nil {
must = append(must, clusterFilter) must = append(must, clusterFilter)
} }
query := util.MapStr{ query := util.MapStr{
"aggs": aggs, "aggs": aggs,
"size": reqBody.Size, "size": reqBody.Size,
@ -190,7 +185,7 @@ func (h *APIHandler) SearchNodeMetadata(w http.ResponseWriter, req *http.Request
response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(orm.GetIndexName(elastic.NodeConfig{}), dsl) response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(orm.GetIndexName(elastic.NodeConfig{}), dsl)
if err != nil { if err != nil {
resBody["error"] = err.Error() resBody["error"] = err.Error()
h.WriteJSON(w,resBody, http.StatusInternalServerError ) h.WriteJSON(w, resBody, http.StatusInternalServerError)
return return
} }
w.Write(util.MustToJSONBytes(response)) w.Write(util.MustToJSONBytes(response))
@ -317,10 +312,10 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps
return return
} }
// 索引速率 // 索引速率
indexMetric:=newMetricItem("indexing", 1, OperationGroupKey) indexMetric := newMetricItem("indexing", 1, OperationGroupKey)
indexMetric.AddAxi("indexing rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) indexMetric.AddAxi("indexing rate", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems := []GroupMetricItem{} nodeMetricItems := []GroupMetricItem{}
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "indexing", Key: "indexing",
Field: "payload.elasticsearch.node_stats.indices.indexing.index_total", Field: "payload.elasticsearch.node_stats.indices.indexing.index_total",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -329,9 +324,9 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps
FormatType: "num", FormatType: "num",
Units: "Indexing/s", Units: "Indexing/s",
}) })
queryMetric:=newMetricItem("search", 2, OperationGroupKey) queryMetric := newMetricItem("search", 2, OperationGroupKey)
queryMetric.AddAxi("query rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) queryMetric.AddAxi("query rate", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "search", Key: "search",
Field: "payload.elasticsearch.node_stats.indices.search.query_total", Field: "payload.elasticsearch.node_stats.indices.search.query_total",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -346,9 +341,9 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps
bucketSize = 60 bucketSize = 60
} }
var metricLen = 15 var metricLen = 15
aggs:=map[string]interface{}{} aggs := map[string]interface{}{}
query=map[string]interface{}{} query = map[string]interface{}{}
query["query"]=util.MapStr{ query["query"] = util.MapStr{
"bool": util.MapStr{ "bool": util.MapStr{
"must": []util.MapStr{ "must": []util.MapStr{
{ {
@ -375,7 +370,7 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps
{ {
"range": util.MapStr{ "range": util.MapStr{
"timestamp": util.MapStr{ "timestamp": util.MapStr{
"gte": fmt.Sprintf("now-%ds", metricLen * bucketSize), "gte": fmt.Sprintf("now-%ds", metricLen*bucketSize),
}, },
}, },
}, },
@ -383,15 +378,15 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps
}, },
} }
for _,metricItem:=range nodeMetricItems{ for _, metricItem := range nodeMetricItems {
aggs[metricItem.ID]=util.MapStr{ aggs[metricItem.ID] = util.MapStr{
"max":util.MapStr{ "max": util.MapStr{
"field": metricItem.Field, "field": metricItem.Field,
}, },
} }
if metricItem.IsDerivative{ if metricItem.IsDerivative {
aggs[metricItem.ID+"_deriv"]=util.MapStr{ aggs[metricItem.ID+"_deriv"] = util.MapStr{
"derivative":util.MapStr{ "derivative": util.MapStr{
"buckets_path": metricItem.ID, "buckets_path": metricItem.ID,
}, },
} }
@ -403,8 +398,8 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps
if err != nil { if err != nil {
panic(err) panic(err)
} }
query["size"]=0 query["size"] = 0
query["aggs"]= util.MapStr{ query["aggs"] = util.MapStr{
"group_by_level": util.MapStr{ "group_by_level": util.MapStr{
"terms": util.MapStr{ "terms": util.MapStr{
"field": "metadata.labels.node_id", "field": "metadata.labels.node_id",
@ -412,11 +407,11 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps
}, },
"aggs": util.MapStr{ "aggs": util.MapStr{
"dates": util.MapStr{ "dates": util.MapStr{
"date_histogram":util.MapStr{ "date_histogram": util.MapStr{
"field": "timestamp", "field": "timestamp",
intervalField: bucketSizeStr, intervalField: bucketSizeStr,
}, },
"aggs":aggs, "aggs": aggs,
}, },
}, },
}, },
@ -430,9 +425,8 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps
indexMetrics := map[string]util.MapStr{} indexMetrics := map[string]util.MapStr{}
for key, item := range metrics { for key, item := range metrics {
for _, line := range item.Lines { for _, line := range item.Lines {
if _, ok := indexMetrics[line.Metric.Label]; !ok{ if _, ok := indexMetrics[line.Metric.Label]; !ok {
indexMetrics[line.Metric.Label] = util.MapStr{ indexMetrics[line.Metric.Label] = util.MapStr{}
}
} }
indexMetrics[line.Metric.Label][key] = line.Data indexMetrics[line.Metric.Label][key] = line.Data
} }
@ -518,7 +512,7 @@ func (h *APIHandler) GetNodeInfo(w http.ResponseWriter, req *http.Request, ps ht
tt, _ := time.Parse(time.RFC3339, ts) tt, _ := time.Parse(time.RFC3339, ts)
if time.Now().Sub(tt).Seconds() > 30 { if time.Now().Sub(tt).Seconds() > 30 {
kvs["status"] = "unavailable" kvs["status"] = "unavailable"
}else{ } else {
kvs["status"] = "available" kvs["status"] = "available"
} }
} }
@ -536,7 +530,7 @@ func (h *APIHandler) GetNodeInfo(w http.ResponseWriter, req *http.Request, ps ht
jvm, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "jvm"}, vresult) jvm, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "jvm"}, vresult)
if ok { if ok {
if jvmVal, ok := jvm.(map[string]interface{});ok { if jvmVal, ok := jvm.(map[string]interface{}); ok {
kvs["jvm"] = util.MapStr{ kvs["jvm"] = util.MapStr{
"mem": jvmVal["mem"], "mem": jvmVal["mem"],
"uptime": jvmVal["uptime_in_millis"], "uptime": jvmVal["uptime_in_millis"],
@ -559,7 +553,7 @@ func (h *APIHandler) GetNodeInfo(w http.ResponseWriter, req *http.Request, ps ht
} }
} }
} }
if len( response.Hits.Hits) > 0 { if len(response.Hits.Hits) > 0 {
hit := response.Hits.Hits[0] hit := response.Hits.Hits[0]
innerMetaData, _ := util.GetMapValueByKeys([]string{"metadata", "labels"}, hit.Source) innerMetaData, _ := util.GetMapValueByKeys([]string{"metadata", "labels"}, hit.Source)
if mp, ok := innerMetaData.(map[string]interface{}); ok { if mp, ok := innerMetaData.(map[string]interface{}); ok {
@ -593,15 +587,15 @@ func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Reque
} }
should := []util.MapStr{ should := []util.MapStr{
{ {
"term":util.MapStr{ "term": util.MapStr{
"metadata.labels.cluster_id":util.MapStr{ "metadata.labels.cluster_id": util.MapStr{
"value": clusterID, "value": clusterID,
}, },
}, },
}, },
{ {
"term":util.MapStr{ "term": util.MapStr{
"metadata.labels.cluster_uuid":util.MapStr{ "metadata.labels.cluster_uuid": util.MapStr{
"value": clusterUUID, "value": clusterUUID,
}, },
}, },
@ -632,15 +626,15 @@ func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Reque
}, },
} }
resBody := map[string]interface{}{} resBody := map[string]interface{}{}
bucketSize, min, max, err := h.GetMetricRangeAndBucketSize(req,clusterID, v1.MetricTypeNodeStats,60) bucketSize, min, max, err := h.GetMetricRangeAndBucketSize(req, clusterID, v1.MetricTypeNodeStats, 60)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
resBody["error"] = err resBody["error"] = err
h.WriteJSON(w, resBody, http.StatusInternalServerError) h.WriteJSON(w, resBody, http.StatusInternalServerError)
return return
} }
query:=map[string]interface{}{} query := map[string]interface{}{}
query["query"]=util.MapStr{ query["query"] = util.MapStr{
"bool": util.MapStr{ "bool": util.MapStr{
"must": must, "must": must,
"minimum_should_match": 1, "minimum_should_match": 1,
@ -658,8 +652,8 @@ func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Reque
}, },
} }
bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
metricItems:=[]*common.MetricItem{} metricItems := []*common.MetricItem{}
metricKey := h.GetParameter(req, "key") metricKey := h.GetParameter(req, "key")
timeout := h.GetParameterOrDefault(req, "timeout", "60s") timeout := h.GetParameterOrDefault(req, "timeout", "60s")
du, err := time.ParseDuration(timeout) du, err := time.ParseDuration(timeout)
@ -679,7 +673,7 @@ func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Reque
return return
} }
metrics["node_health"] = healthMetric metrics["node_health"] = healthMetric
}else if metricKey == ShardStateMetricKey { } else if metricKey == ShardStateMetricKey {
query = util.MapStr{ query = util.MapStr{
"size": 0, "size": 0,
"query": util.MapStr{ "query": util.MapStr{
@ -729,74 +723,74 @@ func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Reque
return return
} }
metrics["shard_state"] = shardStateMetric metrics["shard_state"] = shardStateMetric
}else{ } else {
switch metricKey { switch metricKey {
case NodeProcessCPUMetricKey: case NodeProcessCPUMetricKey:
metricItem:=newMetricItem("cpu", 1, SystemGroupKey) metricItem := newMetricItem("cpu", 1, SystemGroupKey)
metricItem.AddAxi("cpu","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true) metricItem.AddAxi("cpu", "group1", common.PositionLeft, "ratio", "0.[0]", "0.[0]", 5, true)
metricItem.AddLine("Process CPU","Process CPU","process cpu used percent of node.","group1","payload.elasticsearch.node_stats.process.cpu.percent","max",bucketSizeStr,"%","num","0,0.[00]","0,0.[00]",false,false) metricItem.AddLine("Process CPU", "Process CPU", "process cpu used percent of node.", "group1", "payload.elasticsearch.node_stats.process.cpu.percent", "max", bucketSizeStr, "%", "num", "0,0.[00]", "0,0.[00]", false, false)
metricItem.AddLine("OS CPU","OS CPU","process cpu used percent of node.","group1","payload.elasticsearch.node_stats.os.cpu.percent","max",bucketSizeStr,"%","num","0,0.[00]","0,0.[00]",false,false) metricItem.AddLine("OS CPU", "OS CPU", "process cpu used percent of node.", "group1", "payload.elasticsearch.node_stats.os.cpu.percent", "max", bucketSizeStr, "%", "num", "0,0.[00]", "0,0.[00]", false, false)
metricItems=append(metricItems,metricItem) metricItems = append(metricItems, metricItem)
case NodeCPUJVMMetricKey: case NodeCPUJVMMetricKey:
metricItem := newMetricItem("jvm", 2, SystemGroupKey) metricItem := newMetricItem("jvm", 2, SystemGroupKey)
metricItem.AddAxi("JVM Heap","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) metricItem.AddAxi("JVM Heap", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
metricItem.AddLine("Max Heap","Max Heap","JVM max Heap of node.","group1","payload.elasticsearch.node_stats.jvm.mem.heap_max_in_bytes","max",bucketSizeStr,"","bytes","0,0.[00]","0,0.[00]",false,false) metricItem.AddLine("Max Heap", "Max Heap", "JVM max Heap of node.", "group1", "payload.elasticsearch.node_stats.jvm.mem.heap_max_in_bytes", "max", bucketSizeStr, "", "bytes", "0,0.[00]", "0,0.[00]", false, false)
metricItem.AddLine("Used Heap","Used Heap","JVM used Heap of node.","group1","payload.elasticsearch.node_stats.jvm.mem.heap_used_in_bytes","max",bucketSizeStr,"","bytes","0,0.[00]","0,0.[00]",false,false) metricItem.AddLine("Used Heap", "Used Heap", "JVM used Heap of node.", "group1", "payload.elasticsearch.node_stats.jvm.mem.heap_used_in_bytes", "max", bucketSizeStr, "", "bytes", "0,0.[00]", "0,0.[00]", false, false)
metricItems=append(metricItems,metricItem) metricItems = append(metricItems, metricItem)
case v1.IndexThroughputMetricKey: case v1.IndexThroughputMetricKey:
metricItem := newMetricItem("index_throughput", 3, OperationGroupKey) metricItem := newMetricItem("index_throughput", 3, OperationGroupKey)
metricItem.AddAxi("indexing","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) metricItem.AddAxi("indexing", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
metricItem.AddLine("Indexing Rate","Total Shards","Number of documents being indexed for node.","group1","payload.elasticsearch.node_stats.indices.indexing.index_total","max",bucketSizeStr,"doc/s","num","0,0.[00]","0,0.[00]",false,true) metricItem.AddLine("Indexing Rate", "Total Shards", "Number of documents being indexed for node.", "group1", "payload.elasticsearch.node_stats.indices.indexing.index_total", "max", bucketSizeStr, "doc/s", "num", "0,0.[00]", "0,0.[00]", false, true)
metricItems=append(metricItems,metricItem) metricItems = append(metricItems, metricItem)
case v1.SearchThroughputMetricKey: case v1.SearchThroughputMetricKey:
metricItem := newMetricItem("search_throughput", 4, OperationGroupKey) metricItem := newMetricItem("search_throughput", 4, OperationGroupKey)
metricItem.AddAxi("searching","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,false) metricItem.AddAxi("searching", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
metricItem.AddLine("Search Rate","Total Shards", metricItem.AddLine("Search Rate", "Total Shards",
"Number of search requests being executed.", "Number of search requests being executed.",
"group1","payload.elasticsearch.node_stats.indices.search.query_total","max",bucketSizeStr,"query/s","num","0,0.[00]","0,0.[00]",false,true) "group1", "payload.elasticsearch.node_stats.indices.search.query_total", "max", bucketSizeStr, "query/s", "num", "0,0.[00]", "0,0.[00]", false, true)
metricItems=append(metricItems,metricItem) metricItems = append(metricItems, metricItem)
case v1.IndexLatencyMetricKey: case v1.IndexLatencyMetricKey:
metricItem := newMetricItem("index_latency", 5, LatencyGroupKey) metricItem := newMetricItem("index_latency", 5, LatencyGroupKey)
metricItem.AddAxi("indexing","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) metricItem.AddAxi("indexing", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
metricItem.AddLine("Indexing","Indexing Latency","Average latency for indexing documents.","group1","payload.elasticsearch.node_stats.indices.indexing.index_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) metricItem.AddLine("Indexing", "Indexing Latency", "Average latency for indexing documents.", "group1", "payload.elasticsearch.node_stats.indices.indexing.index_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.node_stats.indices.indexing.index_total" metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.node_stats.indices.indexing.index_total"
metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 { metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
return value/value2 return value / value2
} }
metricItem.AddLine("Indexing","Delete Latency","Average latency for delete documents.","group1","payload.elasticsearch.node_stats.indices.indexing.delete_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) metricItem.AddLine("Indexing", "Delete Latency", "Average latency for delete documents.", "group1", "payload.elasticsearch.node_stats.indices.indexing.delete_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.node_stats.indices.indexing.delete_total" metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.node_stats.indices.indexing.delete_total"
metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 { metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
return value/value2 return value / value2
} }
metricItems=append(metricItems,metricItem) metricItems = append(metricItems, metricItem)
case v1.SearchLatencyMetricKey: case v1.SearchLatencyMetricKey:
metricItem := newMetricItem("search_latency", 6, LatencyGroupKey) metricItem := newMetricItem("search_latency", 6, LatencyGroupKey)
metricItem.AddAxi("searching","group2",common.PositionLeft,"num","0,0","0,0.[00]",5,false) metricItem.AddAxi("searching", "group2", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
metricItem.AddLine("Searching","Query Latency","Average latency for searching query.","group2","payload.elasticsearch.node_stats.indices.search.query_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) metricItem.AddLine("Searching", "Query Latency", "Average latency for searching query.", "group2", "payload.elasticsearch.node_stats.indices.search.query_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.query_total" metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.query_total"
metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 { metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
return value/value2 return value / value2
} }
metricItem.AddLine("Searching","Fetch Latency","Average latency for searching fetch.","group2","payload.elasticsearch.node_stats.indices.search.fetch_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) metricItem.AddLine("Searching", "Fetch Latency", "Average latency for searching fetch.", "group2", "payload.elasticsearch.node_stats.indices.search.fetch_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.fetch_total" metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.fetch_total"
metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 { metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
return value/value2 return value / value2
} }
metricItem.AddLine("Searching","Scroll Latency","Average latency for searching fetch.","group2","payload.elasticsearch.node_stats.indices.search.scroll_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) metricItem.AddLine("Searching", "Scroll Latency", "Average latency for searching fetch.", "group2", "payload.elasticsearch.node_stats.indices.search.scroll_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
metricItem.Lines[2].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.scroll_total" metricItem.Lines[2].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.scroll_total"
metricItem.Lines[2].Metric.Calc = func(value, value2 float64) float64 { metricItem.Lines[2].Metric.Calc = func(value, value2 float64) float64 {
return value/value2 return value / value2
} }
metricItems=append(metricItems,metricItem) metricItems = append(metricItems, metricItem)
case ParentBreakerMetricKey: case ParentBreakerMetricKey:
metricItem := newMetricItem("parent_breaker", 8, SystemGroupKey) metricItem := newMetricItem("parent_breaker", 8, SystemGroupKey)
metricItem.AddLine("Parent Breaker Tripped","Parent Breaker Tripped","Rate of the circuit breaker has been triggered and prevented an out of memory error.","group1","payload.elasticsearch.node_stats.breakers.parent.tripped","max",bucketSizeStr,"times/s","num","0,0.[00]","0,0.[00]",false,true) metricItem.AddLine("Parent Breaker Tripped", "Parent Breaker Tripped", "Rate of the circuit breaker has been triggered and prevented an out of memory error.", "group1", "payload.elasticsearch.node_stats.breakers.parent.tripped", "max", bucketSizeStr, "times/s", "num", "0,0.[00]", "0,0.[00]", false, true)
metricItems=append(metricItems,metricItem) metricItems = append(metricItems, metricItem)
} }
metrics, err = h.getSingleMetrics(ctx, metricItems,query, bucketSize) metrics, err = h.getSingleMetrics(ctx, metricItems, query, bucketSize)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
h.WriteError(w, err, http.StatusInternalServerError) h.WriteError(w, err, http.StatusInternalServerError)
@ -808,7 +802,7 @@ func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Reque
minBucketSize, err := v1.GetMetricMinBucketSize(clusterID, v1.MetricTypeNodeStats) minBucketSize, err := v1.GetMetricMinBucketSize(clusterID, v1.MetricTypeNodeStats)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
}else{ } else {
metrics[metricKey].MinBucketSize = int64(minBucketSize) metrics[metricKey].MinBucketSize = int64(minBucketSize)
} }
} }
@ -818,8 +812,8 @@ func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Reque
h.WriteJSON(w, resBody, http.StatusOK) h.WriteJSON(w, resBody, http.StatusOK)
} }
func getNodeShardStateMetric(ctx context.Context, query util.MapStr, bucketSize int)(*common.MetricItem, error){ func getNodeShardStateMetric(ctx context.Context, query util.MapStr, bucketSize int) (*common.MetricItem, error) {
bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr)
if err != nil { if err != nil {
return nil, err return nil, err
@ -848,8 +842,8 @@ func getNodeShardStateMetric(ctx context.Context, query util.MapStr, bucketSize
return nil, err return nil, err
} }
metricItem:=newMetricItem("shard_state", 0, "") metricItem := newMetricItem("shard_state", 0, "")
metricItem.AddLine("Shard State","Shard State","","group1","payload.elasticsearch.shard_stats.routing.state","count",bucketSizeStr,"","ratio","0.[00]","0.[00]",false,false) metricItem.AddLine("Shard State", "Shard State", "", "group1", "payload.elasticsearch.shard_stats.routing.state", "count", bucketSizeStr, "", "ratio", "0.[00]", "0.[00]", false, false)
metricData := []interface{}{} metricData := []interface{}{}
if response.StatusCode == 200 { if response.StatusCode == 200 {
@ -864,8 +858,8 @@ func getNodeShardStateMetric(ctx context.Context, query util.MapStr, bucketSize
return metricItem, nil return metricItem, nil
} }
func getNodeHealthMetric(ctx context.Context, query util.MapStr, bucketSize int)(*common.MetricItem, error){ func getNodeHealthMetric(ctx context.Context, query util.MapStr, bucketSize int) (*common.MetricItem, error) {
bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr)
if err != nil { if err != nil {
return nil, err return nil, err
@ -892,8 +886,8 @@ func getNodeHealthMetric(ctx context.Context, query util.MapStr, bucketSize int)
return nil, err return nil, err
} }
metricItem:=newMetricItem("node_health", 0, "") metricItem := newMetricItem("node_health", 0, "")
metricItem.AddLine("Node health","Node Health","","group1","payload.elasticsearch.node_stats.jvm.uptime_in_millis","min",bucketSizeStr,"%","ratio","0.[00]","0.[00]",false,false) metricItem.AddLine("Node health", "Node Health", "", "group1", "payload.elasticsearch.node_stats.jvm.uptime_in_millis", "min", bucketSizeStr, "%", "ratio", "0.[00]", "0.[00]", false, false)
metricData := []interface{}{} metricData := []interface{}{}
if response.StatusCode == 200 { if response.StatusCode == 200 {
@ -923,7 +917,7 @@ func getNodeHealthMetric(ctx context.Context, query util.MapStr, bucketSize int)
return metricItem, nil return metricItem, nil
} }
func getNodeOnlineStatusOfRecentDay(nodeIDs []string)(map[string][]interface{}, error){ func getNodeOnlineStatusOfRecentDay(nodeIDs []string) (map[string][]interface{}, error) {
q := orm.Query{ q := orm.Query{
WildcardIndex: true, WildcardIndex: true,
} }
@ -977,7 +971,7 @@ func getNodeOnlineStatusOfRecentDay(nodeIDs []string)(map[string][]interface{},
{ {
"from": "now-4d/d", "from": "now-4d/d",
"to": "now-3d/d", "to": "now-3d/d",
},{ }, {
"from": "now-3d/d", "from": "now-3d/d",
"to": "now-2d/d", "to": "now-2d/d",
}, { }, {
@ -1018,7 +1012,7 @@ func getNodeOnlineStatusOfRecentDay(nodeIDs []string)(map[string][]interface{},
{ {
"range": util.MapStr{ "range": util.MapStr{
"timestamp": util.MapStr{ "timestamp": util.MapStr{
"gte":"now-15d", "gte": "now-15d",
"lte": "now", "lte": "now",
}, },
}, },
@ -1062,7 +1056,7 @@ func getNodeOnlineStatusOfRecentDay(nodeIDs []string)(map[string][]interface{},
//mark node status as offline when uptime less than 10m //mark node status as offline when uptime less than 10m
if v, ok := minUptime.(float64); ok && v >= 600000 { if v, ok := minUptime.(float64); ok && v >= 600000 {
recentStatus[nodeKey] = append(recentStatus[nodeKey], []interface{}{bkVal["key"], "online"}) recentStatus[nodeKey] = append(recentStatus[nodeKey], []interface{}{bkVal["key"], "online"})
}else{ } else {
recentStatus[nodeKey] = append(recentStatus[nodeKey], []interface{}{bkVal["key"], "offline"}) recentStatus[nodeKey] = append(recentStatus[nodeKey], []interface{}{bkVal["key"], "offline"})
} }
} }
@ -1080,10 +1074,10 @@ func (h *APIHandler) getNodeIndices(w http.ResponseWriter, req *http.Request, ps
max = h.GetParameterOrDefault(req, "max", "now") max = h.GetParameterOrDefault(req, "max", "now")
) )
resBody := map[string] interface{}{} resBody := map[string]interface{}{}
id := ps.ByName("id") id := ps.ByName("id")
nodeUUID := ps.ByName("node_id") nodeUUID := ps.ByName("node_id")
q := &orm.Query{ Size: 1} q := &orm.Query{Size: 1}
q.AddSort("timestamp", orm.DESC) q.AddSort("timestamp", orm.DESC)
q.Conds = orm.And( q.Conds = orm.And(
orm.Eq("metadata.category", "elasticsearch"), orm.Eq("metadata.category", "elasticsearch"),
@ -1095,16 +1089,16 @@ func (h *APIHandler) getNodeIndices(w http.ResponseWriter, req *http.Request, ps
err, result := orm.Search(event.Event{}, q) err, result := orm.Search(event.Event{}, q)
if err != nil { if err != nil {
resBody["error"] = err.Error() resBody["error"] = err.Error()
h.WriteJSON(w,resBody, http.StatusInternalServerError ) h.WriteJSON(w, resBody, http.StatusInternalServerError)
} }
namesM := util.MapStr{} namesM := util.MapStr{}
if len(result.Result) > 0 { if len(result.Result) > 0 {
if data, ok := result.Result[0].(map[string]interface{}); ok { if data, ok := result.Result[0].(map[string]interface{}); ok {
if routingTable, exists := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_routing_table"}, data); exists { if routingTable, exists := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_routing_table"}, data); exists {
if rows, ok := routingTable.([]interface{}); ok{ if rows, ok := routingTable.([]interface{}); ok {
for _, row := range rows { for _, row := range rows {
if v, ok := row.(map[string]interface{}); ok { if v, ok := row.(map[string]interface{}); ok {
if indexName, ok := v["index"].(string); ok{ if indexName, ok := v["index"].(string); ok {
namesM[indexName] = true namesM[indexName] = true
} }
} }
@ -1114,12 +1108,12 @@ func (h *APIHandler) getNodeIndices(w http.ResponseWriter, req *http.Request, ps
} }
} }
indexNames := make([]interface{}, 0, len(namesM) ) indexNames := make([]interface{}, 0, len(namesM))
for name, _ := range namesM { for name, _ := range namesM {
indexNames = append(indexNames, name) indexNames = append(indexNames, name)
} }
q1 := &orm.Query{ Size: 100} q1 := &orm.Query{Size: 100}
q1.AddSort("timestamp", orm.DESC) q1.AddSort("timestamp", orm.DESC)
q1.Conds = orm.And( q1.Conds = orm.And(
orm.Eq("metadata.category", "elasticsearch"), orm.Eq("metadata.category", "elasticsearch"),
@ -1130,13 +1124,13 @@ func (h *APIHandler) getNodeIndices(w http.ResponseWriter, req *http.Request, ps
err, result = orm.Search(elastic.IndexConfig{}, q1) err, result = orm.Search(elastic.IndexConfig{}, q1)
if err != nil { if err != nil {
resBody["error"] = err.Error() resBody["error"] = err.Error()
h.WriteJSON(w,resBody, http.StatusInternalServerError ) h.WriteJSON(w, resBody, http.StatusInternalServerError)
} }
indices, err := h.getLatestIndices(req, min, max, id, &result) indices, err := h.getLatestIndices(req, min, max, id, &result)
if err != nil { if err != nil {
resBody["error"] = err.Error() resBody["error"] = err.Error()
h.WriteJSON(w,resBody, http.StatusInternalServerError ) h.WriteJSON(w, resBody, http.StatusInternalServerError)
} }
h.WriteJSON(w, indices, http.StatusOK) h.WriteJSON(w, indices, http.StatusOK)
@ -1152,6 +1146,7 @@ type ShardsSummary struct {
PriStoreInBytes int64 `json:"pri_store_in_bytes"` PriStoreInBytes int64 `json:"pri_store_in_bytes"`
Timestamp interface{} `json:"timestamp"` Timestamp interface{} `json:"timestamp"`
} }
func (h *APIHandler) getLatestIndices(req *http.Request, min string, max string, clusterID string, result *orm.Result) ([]interface{}, error) { func (h *APIHandler) getLatestIndices(req *http.Request, min string, max string, clusterID string, result *orm.Result) ([]interface{}, error) {
//filter indices //filter indices
allowedIndices, hasAllPrivilege := h.GetAllowedIndices(req, clusterID) allowedIndices, hasAllPrivilege := h.GetAllowedIndices(req, clusterID)
@ -1165,7 +1160,7 @@ func (h *APIHandler) getLatestIndices(req *http.Request, min string, max string,
query := util.MapStr{ query := util.MapStr{
"size": 10000, "size": 10000,
"_source": []string{"metadata.labels.index_name", "payload.elasticsearch.shard_stats.docs","payload.elasticsearch.shard_stats.store", "payload.elasticsearch.shard_stats.routing", "timestamp"}, "_source": []string{"metadata.labels.index_name", "payload.elasticsearch.shard_stats.docs", "payload.elasticsearch.shard_stats.store", "payload.elasticsearch.shard_stats.routing", "timestamp"},
"collapse": util.MapStr{ "collapse": util.MapStr{
"field": "metadata.labels.shard_id", "field": "metadata.labels.shard_id",
}, },
@ -1240,7 +1235,7 @@ func (h *APIHandler) getLatestIndices(req *http.Request, min string, max string,
} }
if primary == true { if primary == true {
indexInfo.Shards++ indexInfo.Shards++
}else{ } else {
indexInfo.Replicas++ indexInfo.Replicas++
} }
indexInfo.Timestamp = hitM["timestamp"] indexInfo.Timestamp = hitM["timestamp"]
@ -1249,7 +1244,7 @@ func (h *APIHandler) getLatestIndices(req *http.Request, min string, max string,
} }
indices := []interface{}{} indices := []interface{}{}
var indexPattern *radix.Pattern var indexPattern *radix.Pattern
if !hasAllPrivilege{ if !hasAllPrivilege {
indexPattern = radix.Compile(allowedIndices...) indexPattern = radix.Compile(allowedIndices...)
} }
@ -1280,7 +1275,7 @@ func (h *APIHandler) getLatestIndices(req *http.Request, min string, max string,
"docs_count": indexInfos[v].DocsCount, "docs_count": indexInfos[v].DocsCount,
"shards": indexInfos[v].Shards, "shards": indexInfos[v].Shards,
"replicas": replicasNum, "replicas": replicasNum,
"unassigned_shards": (replicasNum + 1) * shardsNum - indexInfos[v].Shards - replicasNum, "unassigned_shards": (replicasNum+1)*shardsNum - indexInfos[v].Shards - replicasNum,
"store_size": util.FormatBytes(float64(indexInfos[v].StoreInBytes), 1), "store_size": util.FormatBytes(float64(indexInfos[v].StoreInBytes), 1),
}) })
} else { } else {
@ -1297,7 +1292,6 @@ func (h *APIHandler) getLatestIndices(req *http.Request, min string, max string,
return indices, nil return indices, nil
} }
func (h *APIHandler) GetNodeShards(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *APIHandler) GetNodeShards(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
clusterID := ps.MustGetParameter("id") clusterID := ps.MustGetParameter("id")
if GetMonitorState(clusterID) == elastic.ModeAgentless { if GetMonitorState(clusterID) == elastic.ModeAgentless {
@ -1327,7 +1321,7 @@ func (h *APIHandler) GetNodeShards(w http.ResponseWriter, req *http.Request, ps
err, result := orm.Search(&event.Event{}, &q1) err, result := orm.Search(&event.Event{}, &q1)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
h.WriteError(w, err.Error(), http.StatusInternalServerError ) h.WriteError(w, err.Error(), http.StatusInternalServerError)
return return
} }
var shards = []interface{}{} var shards = []interface{}{}
@ -1360,7 +1354,7 @@ func (h *APIHandler) GetNodeShards(w http.ResponseWriter, req *http.Request, ps
primary, _ := shardM.GetValue("routing.primary") primary, _ := shardM.GetValue("routing.primary")
if primary == true { if primary == true {
shardInfo["prirep"] = "p" shardInfo["prirep"] = "p"
}else{ } else {
shardInfo["prirep"] = "r" shardInfo["prirep"] = "r"
} }
shardInfo["state"], _ = shardM.GetValue("routing.state") shardInfo["state"], _ = shardM.GetValue("routing.state")
@ -1380,7 +1374,7 @@ func (h *APIHandler) GetNodeShards(w http.ResponseWriter, req *http.Request, ps
h.WriteJSON(w, shards, http.StatusOK) h.WriteJSON(w, shards, http.StatusOK)
} }
//deleteNodeMetadata used to clean node metadata after node is offline and not active within 7 days // deleteNodeMetadata used to clean node metadata after node is offline and not active within 7 days
func (h APIHandler) deleteNodeMetadata(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h APIHandler) deleteNodeMetadata(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
indexName := orm.GetIndexName(elastic.NodeConfig{}) indexName := orm.GetIndexName(elastic.NodeConfig{})

2
modules/elastic/api/proxy.go
View File

@ -278,5 +278,3 @@ func rewriteTableNamesOfSqlRequest(req *http.Request, distribution string) (stri
} }
return strings.Join(unescapedTableNames, ","), nil return strings.Join(unescapedTableNames, ","), nil
} }

91
modules/elastic/api/search.go
View File

@ -38,11 +38,10 @@ import (
"time" "time"
) )
func (h *APIHandler) HandleCreateSearchTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ func (h *APIHandler) HandleCreateSearchTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody := map[string] interface{}{ resBody := map[string]interface{}{}
}
targetClusterID := ps.ByName("id") targetClusterID := ps.ByName("id")
exists,client,err:=h.GetClusterClient(targetClusterID) exists, client, err := h.GetClusterClient(targetClusterID)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
@ -51,8 +50,8 @@ func (h *APIHandler) HandleCreateSearchTemplateAction(w http.ResponseWriter, req
return return
} }
if !exists{ if !exists {
resBody["error"] = fmt.Sprintf("cluster [%s] not found",targetClusterID) resBody["error"] = fmt.Sprintf("cluster [%s] not found", targetClusterID)
log.Error(resBody["error"]) log.Error(resBody["error"])
h.WriteJSON(w, resBody, http.StatusNotFound) h.WriteJSON(w, resBody, http.StatusNotFound)
return return
@ -89,7 +88,7 @@ func (h *APIHandler) HandleCreateSearchTemplateAction(w http.ResponseWriter, req
template.Created = time.Now() template.Created = time.Now()
template.Updated = template.Created template.Updated = template.Created
template.ClusterID = targetClusterID template.ClusterID = targetClusterID
index:=orm.GetIndexName(elastic.SearchTemplate{}) index := orm.GetIndexName(elastic.SearchTemplate{})
insertRes, err := esClient.Index(index, "", id, template, "wait_for") insertRes, err := esClient.Index(index, "", id, template, "wait_for")
if err != nil { if err != nil {
log.Error(err) log.Error(err)
@ -102,14 +101,13 @@ func (h *APIHandler) HandleCreateSearchTemplateAction(w http.ResponseWriter, req
resBody["_id"] = id resBody["_id"] = id
resBody["result"] = insertRes.Result resBody["result"] = insertRes.Result
h.WriteJSON(w, resBody,http.StatusOK) h.WriteJSON(w, resBody, http.StatusOK)
} }
func (h *APIHandler) HandleUpdateSearchTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ func (h *APIHandler) HandleUpdateSearchTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody := map[string] interface{}{ resBody := map[string]interface{}{}
}
targetClusterID := ps.ByName("id") targetClusterID := ps.ByName("id")
exists,client,err:=h.GetClusterClient(targetClusterID) exists, client, err := h.GetClusterClient(targetClusterID)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
@ -118,8 +116,8 @@ func (h *APIHandler) HandleUpdateSearchTemplateAction(w http.ResponseWriter, req
return return
} }
if !exists{ if !exists {
resBody["error"] = fmt.Sprintf("cluster [%s] not found",targetClusterID) resBody["error"] = fmt.Sprintf("cluster [%s] not found", targetClusterID)
log.Error(resBody["error"]) log.Error(resBody["error"])
h.WriteJSON(w, resBody, http.StatusNotFound) h.WriteJSON(w, resBody, http.StatusNotFound)
return return
@ -136,8 +134,8 @@ func (h *APIHandler) HandleUpdateSearchTemplateAction(w http.ResponseWriter, req
} }
templateID := ps.ByName("template_id") templateID := ps.ByName("template_id")
esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
index:=orm.GetIndexName(elastic.SearchTemplate{}) index := orm.GetIndexName(elastic.SearchTemplate{})
getRes, err := esClient.Get(index, "",templateID) getRes, err := esClient.Get(index, "", templateID)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
resBody["error"] = err.Error() resBody["error"] = err.Error()
@ -207,14 +205,13 @@ func (h *APIHandler) HandleUpdateSearchTemplateAction(w http.ResponseWriter, req
resBody["_id"] = templateID resBody["_id"] = templateID
resBody["result"] = insertRes.Result resBody["result"] = insertRes.Result
h.WriteJSON(w, resBody,http.StatusOK) h.WriteJSON(w, resBody, http.StatusOK)
} }
func (h *APIHandler) HandleDeleteSearchTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ func (h *APIHandler) HandleDeleteSearchTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody := map[string] interface{}{ resBody := map[string]interface{}{}
}
targetClusterID := ps.ByName("id") targetClusterID := ps.ByName("id")
exists,client,err:=h.GetClusterClient(targetClusterID) exists, client, err := h.GetClusterClient(targetClusterID)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
resBody["error"] = err.Error() resBody["error"] = err.Error()
@ -222,8 +219,8 @@ func (h *APIHandler) HandleDeleteSearchTemplateAction(w http.ResponseWriter, req
return return
} }
if !exists{ if !exists {
resBody["error"] = fmt.Sprintf("cluster [%s] not found",targetClusterID) resBody["error"] = fmt.Sprintf("cluster [%s] not found", targetClusterID)
log.Error(resBody["error"]) log.Error(resBody["error"])
h.WriteJSON(w, resBody, http.StatusNotFound) h.WriteJSON(w, resBody, http.StatusNotFound)
return return
@ -231,7 +228,7 @@ func (h *APIHandler) HandleDeleteSearchTemplateAction(w http.ResponseWriter, req
templateID := ps.ByName("template_id") templateID := ps.ByName("template_id")
index:=orm.GetIndexName(elastic.SearchTemplate{}) index := orm.GetIndexName(elastic.SearchTemplate{})
esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
res, err := esClient.Get(index, "", templateID) res, err := esClient.Get(index, "", templateID)
if err != nil { if err != nil {
@ -273,9 +270,8 @@ func (h *APIHandler) HandleDeleteSearchTemplateAction(w http.ResponseWriter, req
} }
func (h *APIHandler) HandleSearchSearchTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ func (h *APIHandler) HandleSearchSearchTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody := map[string] interface{}{ resBody := map[string]interface{}{}
}
var ( var (
name = h.GetParameterOrDefault(req, "name", "") name = h.GetParameterOrDefault(req, "name", "")
strFrom = h.GetParameterOrDefault(req, "from", "0") strFrom = h.GetParameterOrDefault(req, "from", "0")
@ -287,7 +283,7 @@ func (h *APIHandler) HandleSearchSearchTemplateAction(w http.ResponseWriter, req
size, _ := strconv.Atoi(strSize) size, _ := strconv.Atoi(strSize)
targetClusterID := ps.ByName("id") targetClusterID := ps.ByName("id")
mustBuilder.WriteString(fmt.Sprintf(`{"match":{"cluster_id": "%s"}}`, targetClusterID)) mustBuilder.WriteString(fmt.Sprintf(`{"match":{"cluster_id": "%s"}}`, targetClusterID))
if name != ""{ if name != "" {
mustBuilder.WriteString(fmt.Sprintf(`,{"match":{"name": "%s"}}`, name)) mustBuilder.WriteString(fmt.Sprintf(`,{"match":{"name": "%s"}}`, name))
} }
@ -305,8 +301,8 @@ func (h *APIHandler) HandleSearchSearchTemplateAction(w http.ResponseWriter, req
h.WriteJSON(w, res, http.StatusOK) h.WriteJSON(w, res, http.StatusOK)
} }
func (h *APIHandler) HandleGetSearchTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ func (h *APIHandler) HandleGetSearchTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody := map[string] interface{}{} resBody := map[string]interface{}{}
id := ps.ByName("template_id") id := ps.ByName("template_id")
indexName := orm.GetIndexName(elastic.SearchTemplate{}) indexName := orm.GetIndexName(elastic.SearchTemplate{})
@ -314,19 +310,18 @@ func (h *APIHandler) HandleGetSearchTemplateAction(w http.ResponseWriter, req *h
if err != nil { if err != nil {
log.Error(err) log.Error(err)
resBody["error"] = err.Error() resBody["error"] = err.Error()
if getResponse!=nil{ if getResponse != nil {
h.WriteJSON(w, resBody, getResponse.StatusCode) h.WriteJSON(w, resBody, getResponse.StatusCode)
}else{ } else {
h.WriteJSON(w, resBody, http.StatusInternalServerError) h.WriteJSON(w, resBody, http.StatusInternalServerError)
} }
return return
} }
h.WriteJSON(w,getResponse,200) h.WriteJSON(w, getResponse, 200)
} }
func (h *APIHandler) HandleSearchSearchTemplateHistoryAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ func (h *APIHandler) HandleSearchSearchTemplateHistoryAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody := map[string] interface{}{ resBody := map[string]interface{}{}
}
var ( var (
templateID = h.GetParameterOrDefault(req, "template_id", "") templateID = h.GetParameterOrDefault(req, "template_id", "")
strFrom = h.GetParameterOrDefault(req, "from", "0") strFrom = h.GetParameterOrDefault(req, "from", "0")
@ -338,7 +333,7 @@ func (h *APIHandler) HandleSearchSearchTemplateHistoryAction(w http.ResponseWrit
size, _ := strconv.Atoi(strSize) size, _ := strconv.Atoi(strSize)
targetClusterID := ps.ByName("id") targetClusterID := ps.ByName("id")
mustBuilder.WriteString(fmt.Sprintf(`{"match":{"content.cluster_id": "%s"}}`, targetClusterID)) mustBuilder.WriteString(fmt.Sprintf(`{"match":{"content.cluster_id": "%s"}}`, targetClusterID))
if templateID != ""{ if templateID != "" {
mustBuilder.WriteString(fmt.Sprintf(`,{"match":{"template_id": "%s"}}`, templateID)) mustBuilder.WriteString(fmt.Sprintf(`,{"match":{"template_id": "%s"}}`, templateID))
} }
@ -356,11 +351,10 @@ func (h *APIHandler) HandleSearchSearchTemplateHistoryAction(w http.ResponseWrit
h.WriteJSON(w, res, http.StatusOK) h.WriteJSON(w, res, http.StatusOK)
} }
func (h *APIHandler) HandleRenderTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ func (h *APIHandler) HandleRenderTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody := map[string] interface{}{ resBody := map[string]interface{}{}
}
targetClusterID := ps.ByName("id") targetClusterID := ps.ByName("id")
exists,client,err:=h.GetClusterClient(targetClusterID) exists, client, err := h.GetClusterClient(targetClusterID)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
resBody["error"] = err.Error() resBody["error"] = err.Error()
@ -368,8 +362,8 @@ func (h *APIHandler) HandleRenderTemplateAction(w http.ResponseWriter, req *http
return return
} }
if !exists{ if !exists {
resBody["error"] = fmt.Sprintf("cluster [%s] not found",targetClusterID) resBody["error"] = fmt.Sprintf("cluster [%s] not found", targetClusterID)
log.Error(resBody["error"]) log.Error(resBody["error"])
h.WriteJSON(w, resBody, http.StatusNotFound) h.WriteJSON(w, resBody, http.StatusNotFound)
return return
@ -394,11 +388,10 @@ func (h *APIHandler) HandleRenderTemplateAction(w http.ResponseWriter, req *http
h.WriteJSON(w, string(res), http.StatusOK) h.WriteJSON(w, string(res), http.StatusOK)
} }
func (h *APIHandler) HandleSearchTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ func (h *APIHandler) HandleSearchTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody := map[string] interface{}{ resBody := map[string]interface{}{}
}
targetClusterID := ps.ByName("id") targetClusterID := ps.ByName("id")
exists,client,err:=h.GetClusterClient(targetClusterID) exists, client, err := h.GetClusterClient(targetClusterID)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
resBody["error"] = err.Error() resBody["error"] = err.Error()
@ -406,8 +399,8 @@ func (h *APIHandler) HandleSearchTemplateAction(w http.ResponseWriter, req *http
return return
} }
if !exists{ if !exists {
resBody["error"] = fmt.Sprintf("cluster [%s] not found",targetClusterID) resBody["error"] = fmt.Sprintf("cluster [%s] not found", targetClusterID)
log.Error(resBody["error"]) log.Error(resBody["error"])
h.WriteJSON(w, resBody, http.StatusNotFound) h.WriteJSON(w, resBody, http.StatusNotFound)
return return

15
modules/elastic/api/setting.go
View File

@ -36,8 +36,7 @@ import (
) )
func (h *APIHandler) HandleSettingAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *APIHandler) HandleSettingAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody := map[string]interface{}{ resBody := map[string]interface{}{}
}
targetClusterID := ps.ByName("id") targetClusterID := ps.ByName("id")
esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
@ -58,12 +57,11 @@ func (h *APIHandler) HandleSettingAction(w http.ResponseWriter, req *http.Reques
searchRes, err := esClient.SearchWithRawQueryDSL(indexName, []byte(queryDSL)) searchRes, err := esClient.SearchWithRawQueryDSL(indexName, []byte(queryDSL))
if len(searchRes.Hits.Hits) > 0 { if len(searchRes.Hits.Hits) > 0 {
_, err = esClient.Index(indexName, "", searchRes.Hits.Hits[0].ID, reqParams, "wait_for") _, err = esClient.Index(indexName, "", searchRes.Hits.Hits[0].ID, reqParams, "wait_for")
}else{ } else {
reqParams.ID = util.GetUUID() reqParams.ID = util.GetUUID()
_, err = esClient.Index(indexName, "", reqParams.ID, reqParams, "wait_for") _, err = esClient.Index(indexName, "", reqParams.ID, reqParams, "wait_for")
} }
if err != nil { if err != nil {
log.Error(err) log.Error(err)
resBody["error"] = err resBody["error"] = err
@ -71,12 +69,11 @@ func (h *APIHandler) HandleSettingAction(w http.ResponseWriter, req *http.Reques
return return
} }
resBody["acknowledged"] = true resBody["acknowledged"] = true
h.WriteJSON(w, resBody ,http.StatusOK) h.WriteJSON(w, resBody, http.StatusOK)
} }
func (h *APIHandler) HandleGetSettingAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *APIHandler) HandleGetSettingAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody := map[string]interface{}{ resBody := map[string]interface{}{}
}
targetClusterID := ps.ByName("id") targetClusterID := ps.ByName("id")
esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
@ -94,8 +91,8 @@ func (h *APIHandler) HandleGetSettingAction(w http.ResponseWriter, req *http.Req
var value interface{} var value interface{}
if len(searchRes.Hits.Hits) > 0 { if len(searchRes.Hits.Hits) > 0 {
value = searchRes.Hits.Hits[0].Source["value"] value = searchRes.Hits.Hits[0].Source["value"]
}else{ } else {
value = "" value = ""
} }
h.WriteJSON(w, value ,http.StatusOK) h.WriteJSON(w, value, http.StatusOK)
} }

4
modules/elastic/api/shard.go
View File

@ -28,12 +28,12 @@
package api package api
import ( import (
log "github.com/cihub/seelog"
httprouter "infini.sh/framework/core/api/router"
"infini.sh/framework/core/event" "infini.sh/framework/core/event"
"infini.sh/framework/core/orm" "infini.sh/framework/core/orm"
"infini.sh/framework/modules/elastic/adapter" "infini.sh/framework/modules/elastic/adapter"
"net/http" "net/http"
log "github.com/cihub/seelog"
httprouter "infini.sh/framework/core/api/router"
) )
func (h *APIHandler) GetShardInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *APIHandler) GetShardInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {

4
modules/elastic/api/template.go
View File

@ -36,7 +36,7 @@ import (
"src/github.com/buger/jsonparser" "src/github.com/buger/jsonparser"
) )
func (h *APIHandler) HandleGetTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ func (h *APIHandler) HandleGetTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
clusterID := ps.MustGetParameter("id") clusterID := ps.MustGetParameter("id")
esClient := elastic.GetClient(clusterID) esClient := elastic.GetClient(clusterID)
templates, err := esClient.GetTemplate("") templates, err := esClient.GetTemplate("")
@ -48,7 +48,7 @@ func (h *APIHandler) HandleGetTemplateAction(w http.ResponseWriter, req *http.Re
h.WriteJSON(w, templates, http.StatusOK) h.WriteJSON(w, templates, http.StatusOK)
} }
func (h *APIHandler) HandleSaveTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ func (h *APIHandler) HandleSaveTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
clusterID := ps.MustGetParameter("id") clusterID := ps.MustGetParameter("id")
templateName := ps.MustGetParameter("template_name") templateName := ps.MustGetParameter("template_name")
esClient := elastic.GetClient(clusterID) esClient := elastic.GetClient(clusterID)

86
modules/elastic/api/threadpool_metrics.go
View File

@ -80,12 +80,12 @@ const (
ForceMergeQueueMetricKey = "force_merge_queue" ForceMergeQueueMetricKey = "force_merge_queue"
) )
func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string, bucketSize int, min, max int64, nodeName string, top int, metricKey string) (map[string]*common.MetricItem, error){ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string, bucketSize int, min, max int64, nodeName string, top int, metricKey string) (map[string]*common.MetricItem, error) {
clusterUUID, err := h.getClusterUUID(clusterID) clusterUUID, err := h.getClusterUUID(clusterID)
if err != nil { if err != nil {
return nil, err return nil, err
} }
bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
var must = []util.MapStr{ var must = []util.MapStr{
{ {
"term": util.MapStr{ "term": util.MapStr{
@ -108,7 +108,7 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
if nodeName != "" { if nodeName != "" {
nodeNames = strings.Split(nodeName, ",") nodeNames = strings.Split(nodeName, ",")
top = len(nodeNames) top = len(nodeNames)
}else{ } else {
nodeNames, err = h.getTopNodeName(clusterID, top, 15) nodeNames, err = h.getTopNodeName(clusterID, top, 15)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
@ -131,7 +131,6 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
}, },
}, },
}, },
}) })
} }
should := []util.MapStr{ should := []util.MapStr{
@ -143,16 +142,16 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
}, },
}, },
{ {
"term":util.MapStr{ "term": util.MapStr{
"metadata.labels.cluster_uuid":util.MapStr{ "metadata.labels.cluster_uuid": util.MapStr{
"value": clusterUUID, "value": clusterUUID,
}, },
}, },
}, },
} }
query:=map[string]interface{}{} query := map[string]interface{}{}
query["query"]=util.MapStr{ query["query"] = util.MapStr{
"bool": util.MapStr{ "bool": util.MapStr{
"must": must, "must": must,
"minimum_should_match": 1, "minimum_should_match": 1,
@ -173,7 +172,7 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
switch metricKey { switch metricKey {
case SearchThreadsMetricKey: case SearchThreadsMetricKey:
searchThreadsMetric := newMetricItem(SearchThreadsMetricKey, 1, ThreadPoolSearchGroupKey) searchThreadsMetric := newMetricItem(SearchThreadsMetricKey, 1, ThreadPoolSearchGroupKey)
searchThreadsMetric.AddAxi("Search Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) searchThreadsMetric.AddAxi("Search Threads Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
queueMetricItems = append(queueMetricItems, GroupMetricItem{ queueMetricItems = append(queueMetricItems, GroupMetricItem{
Key: "search_threads", Key: "search_threads",
Field: "payload.elasticsearch.node_stats.thread_pool.search.threads", Field: "payload.elasticsearch.node_stats.thread_pool.search.threads",
@ -185,7 +184,7 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
}) })
case SearchQueueMetricKey: case SearchQueueMetricKey:
searchQueueMetric := newMetricItem(SearchQueueMetricKey, 1, ThreadPoolSearchGroupKey) searchQueueMetric := newMetricItem(SearchQueueMetricKey, 1, ThreadPoolSearchGroupKey)
searchQueueMetric.AddAxi("Search Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) searchQueueMetric.AddAxi("Search Queue Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
queueMetricItems = append(queueMetricItems, GroupMetricItem{ queueMetricItems = append(queueMetricItems, GroupMetricItem{
Key: "search_queue", Key: "search_queue",
@ -198,7 +197,7 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
}) })
case SearchActiveMetricKey: case SearchActiveMetricKey:
searchActiveMetric := newMetricItem(SearchActiveMetricKey, 1, ThreadPoolSearchGroupKey) searchActiveMetric := newMetricItem(SearchActiveMetricKey, 1, ThreadPoolSearchGroupKey)
searchActiveMetric.AddAxi("Search Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) searchActiveMetric.AddAxi("Search Active Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
queueMetricItems = append(queueMetricItems, GroupMetricItem{ queueMetricItems = append(queueMetricItems, GroupMetricItem{
Key: "search_active", Key: "search_active",
@ -211,7 +210,7 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
}) })
case SearchRejectedMetricKey: case SearchRejectedMetricKey:
searchRejectedMetric := newMetricItem(SearchRejectedMetricKey, 1, ThreadPoolSearchGroupKey) searchRejectedMetric := newMetricItem(SearchRejectedMetricKey, 1, ThreadPoolSearchGroupKey)
searchRejectedMetric.AddAxi("Search Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) searchRejectedMetric.AddAxi("Search Rejected Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
queueMetricItems = append(queueMetricItems, GroupMetricItem{ queueMetricItems = append(queueMetricItems, GroupMetricItem{
Key: "search_rejected", Key: "search_rejected",
@ -224,7 +223,7 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
}) })
case GetThreadsMetricKey: case GetThreadsMetricKey:
getThreadsMetric := newMetricItem(GetThreadsMetricKey, 1, ThreadPoolGetGroupKey) getThreadsMetric := newMetricItem(GetThreadsMetricKey, 1, ThreadPoolGetGroupKey)
getThreadsMetric.AddAxi("Get Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) getThreadsMetric.AddAxi("Get Threads Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
queueMetricItems = append(queueMetricItems, GroupMetricItem{ queueMetricItems = append(queueMetricItems, GroupMetricItem{
Key: "get_threads", Key: "get_threads",
@ -237,7 +236,7 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
}) })
case GetQueueMetricKey: case GetQueueMetricKey:
getQueueMetric := newMetricItem(GetQueueMetricKey, 1, ThreadPoolGetGroupKey) getQueueMetric := newMetricItem(GetQueueMetricKey, 1, ThreadPoolGetGroupKey)
getQueueMetric.AddAxi("Get Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) getQueueMetric.AddAxi("Get Queue Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
queueMetricItems = append(queueMetricItems, GroupMetricItem{ queueMetricItems = append(queueMetricItems, GroupMetricItem{
Key: "get_queue", Key: "get_queue",
@ -250,7 +249,7 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
}) })
case GetActiveMetricKey: case GetActiveMetricKey:
getActiveMetric := newMetricItem(GetActiveMetricKey, 1, ThreadPoolGetGroupKey) getActiveMetric := newMetricItem(GetActiveMetricKey, 1, ThreadPoolGetGroupKey)
getActiveMetric.AddAxi("Get Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) getActiveMetric.AddAxi("Get Active Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
queueMetricItems = append(queueMetricItems, GroupMetricItem{ queueMetricItems = append(queueMetricItems, GroupMetricItem{
Key: "get_active", Key: "get_active",
@ -263,7 +262,7 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
}) })
case GetRejectedMetricKey: case GetRejectedMetricKey:
getRejectedMetric := newMetricItem(GetRejectedMetricKey, 1, ThreadPoolGetGroupKey) getRejectedMetric := newMetricItem(GetRejectedMetricKey, 1, ThreadPoolGetGroupKey)
getRejectedMetric.AddAxi("Get Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) getRejectedMetric.AddAxi("Get Rejected Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
queueMetricItems = append(queueMetricItems, GroupMetricItem{ queueMetricItems = append(queueMetricItems, GroupMetricItem{
Key: "get_rejected", Key: "get_rejected",
@ -276,7 +275,7 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
}) })
case FlushThreadsMetricKey: case FlushThreadsMetricKey:
flushThreadsMetric := newMetricItem(FlushThreadsMetricKey, 1, ThreadPoolFlushGroupKey) flushThreadsMetric := newMetricItem(FlushThreadsMetricKey, 1, ThreadPoolFlushGroupKey)
flushThreadsMetric.AddAxi("Flush Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) flushThreadsMetric.AddAxi("Flush Threads Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
queueMetricItems = append(queueMetricItems, GroupMetricItem{ queueMetricItems = append(queueMetricItems, GroupMetricItem{
Key: "flush_threads", Key: "flush_threads",
@ -289,7 +288,7 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
}) })
case FlushQueueMetricKey: case FlushQueueMetricKey:
flushQueueMetric := newMetricItem(FlushQueueMetricKey, 1, ThreadPoolFlushGroupKey) flushQueueMetric := newMetricItem(FlushQueueMetricKey, 1, ThreadPoolFlushGroupKey)
flushQueueMetric.AddAxi("Get Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) flushQueueMetric.AddAxi("Get Queue Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
queueMetricItems = append(queueMetricItems, GroupMetricItem{ queueMetricItems = append(queueMetricItems, GroupMetricItem{
Key: "flush_queue", Key: "flush_queue",
@ -302,7 +301,7 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
}) })
case FlushActiveMetricKey: case FlushActiveMetricKey:
flushActiveMetric := newMetricItem(FlushActiveMetricKey, 1, ThreadPoolFlushGroupKey) flushActiveMetric := newMetricItem(FlushActiveMetricKey, 1, ThreadPoolFlushGroupKey)
flushActiveMetric.AddAxi("Flush Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) flushActiveMetric.AddAxi("Flush Active Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
queueMetricItems = append(queueMetricItems, GroupMetricItem{ queueMetricItems = append(queueMetricItems, GroupMetricItem{
Key: "flush_active", Key: "flush_active",
@ -316,7 +315,7 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
case FlushRejectedMetricKey: case FlushRejectedMetricKey:
flushRejectedMetric := newMetricItem(FlushRejectedMetricKey, 1, ThreadPoolFlushGroupKey) flushRejectedMetric := newMetricItem(FlushRejectedMetricKey, 1, ThreadPoolFlushGroupKey)
flushRejectedMetric.AddAxi("Flush Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) flushRejectedMetric.AddAxi("Flush Rejected Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
queueMetricItems = append(queueMetricItems, GroupMetricItem{ queueMetricItems = append(queueMetricItems, GroupMetricItem{
Key: "flush_rejected", Key: "flush_rejected",
@ -485,7 +484,7 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
}) })
case RefreshThreadsMetricKey: case RefreshThreadsMetricKey:
refreshThreadsMetric := newMetricItem(RefreshThreadsMetricKey, 1, ThreadPoolRefreshGroupKey) refreshThreadsMetric := newMetricItem(RefreshThreadsMetricKey, 1, ThreadPoolRefreshGroupKey)
refreshThreadsMetric.AddAxi("Refresh Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) refreshThreadsMetric.AddAxi("Refresh Threads Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
queueMetricItems = append(queueMetricItems, GroupMetricItem{ queueMetricItems = append(queueMetricItems, GroupMetricItem{
Key: "refresh_threads", Key: "refresh_threads",
@ -498,7 +497,7 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
}) })
case RefreshQueueMetricKey: case RefreshQueueMetricKey:
refreshQueueMetric := newMetricItem(RefreshQueueMetricKey, 1, ThreadPoolRefreshGroupKey) refreshQueueMetric := newMetricItem(RefreshQueueMetricKey, 1, ThreadPoolRefreshGroupKey)
refreshQueueMetric.AddAxi("Refresh Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) refreshQueueMetric.AddAxi("Refresh Queue Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
queueMetricItems = append(queueMetricItems, GroupMetricItem{ queueMetricItems = append(queueMetricItems, GroupMetricItem{
Key: "refresh_queue", Key: "refresh_queue",
@ -511,7 +510,7 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
}) })
case RefreshActiveMetricKey: case RefreshActiveMetricKey:
refreshActiveMetric := newMetricItem(RefreshActiveMetricKey, 1, ThreadPoolRefreshGroupKey) refreshActiveMetric := newMetricItem(RefreshActiveMetricKey, 1, ThreadPoolRefreshGroupKey)
refreshActiveMetric.AddAxi("Refresh Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) refreshActiveMetric.AddAxi("Refresh Active Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
queueMetricItems = append(queueMetricItems, GroupMetricItem{ queueMetricItems = append(queueMetricItems, GroupMetricItem{
Key: "refresh_active", Key: "refresh_active",
@ -524,7 +523,7 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
}) })
case RefreshRejectedMetricKey: case RefreshRejectedMetricKey:
refreshRejectedMetric := newMetricItem(RefreshRejectedMetricKey, 1, ThreadPoolRefreshGroupKey) refreshRejectedMetric := newMetricItem(RefreshRejectedMetricKey, 1, ThreadPoolRefreshGroupKey)
refreshRejectedMetric.AddAxi("Refresh Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) refreshRejectedMetric.AddAxi("Refresh Rejected Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
queueMetricItems = append(queueMetricItems, GroupMetricItem{ queueMetricItems = append(queueMetricItems, GroupMetricItem{
Key: "refresh_rejected", Key: "refresh_rejected",
@ -537,7 +536,7 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
}) })
case ForceMergeThreadsMetricKey: case ForceMergeThreadsMetricKey:
forceMergeThreadsMetric := newMetricItem(ForceMergeThreadsMetricKey, 1, ThreadPoolForceMergeGroupKey) forceMergeThreadsMetric := newMetricItem(ForceMergeThreadsMetricKey, 1, ThreadPoolForceMergeGroupKey)
forceMergeThreadsMetric.AddAxi("Force Merge Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) forceMergeThreadsMetric.AddAxi("Force Merge Threads Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
queueMetricItems = append(queueMetricItems, GroupMetricItem{ queueMetricItems = append(queueMetricItems, GroupMetricItem{
Key: "force_merge_threads", Key: "force_merge_threads",
@ -550,7 +549,7 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
}) })
case ForceMergeQueueMetricKey: case ForceMergeQueueMetricKey:
forceMergeQueueMetric := newMetricItem(ForceMergeQueueMetricKey, 1, ThreadPoolForceMergeGroupKey) forceMergeQueueMetric := newMetricItem(ForceMergeQueueMetricKey, 1, ThreadPoolForceMergeGroupKey)
forceMergeQueueMetric.AddAxi("Force Merge Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) forceMergeQueueMetric.AddAxi("Force Merge Queue Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
queueMetricItems = append(queueMetricItems, GroupMetricItem{ queueMetricItems = append(queueMetricItems, GroupMetricItem{
Key: "force_merge_queue", Key: "force_merge_queue",
@ -563,7 +562,7 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
}) })
case ForceMergeActiveMetricKey: case ForceMergeActiveMetricKey:
forceMergeActiveMetric := newMetricItem(ForceMergeActiveMetricKey, 1, ThreadPoolForceMergeGroupKey) forceMergeActiveMetric := newMetricItem(ForceMergeActiveMetricKey, 1, ThreadPoolForceMergeGroupKey)
forceMergeActiveMetric.AddAxi("Force Merge Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) forceMergeActiveMetric.AddAxi("Force Merge Active Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
queueMetricItems = append(queueMetricItems, GroupMetricItem{ queueMetricItems = append(queueMetricItems, GroupMetricItem{
Key: "force_merge_active", Key: "force_merge_active",
@ -576,7 +575,7 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
}) })
case ForceMergeRejectedMetricKey: case ForceMergeRejectedMetricKey:
forceMergeRejectedMetric := newMetricItem(ForceMergeRejectedMetricKey, 1, ThreadPoolForceMergeGroupKey) forceMergeRejectedMetric := newMetricItem(ForceMergeRejectedMetricKey, 1, ThreadPoolForceMergeGroupKey)
forceMergeRejectedMetric.AddAxi("Force Merge Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) forceMergeRejectedMetric.AddAxi("Force Merge Rejected Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
queueMetricItems = append(queueMetricItems, GroupMetricItem{ queueMetricItems = append(queueMetricItems, GroupMetricItem{
Key: "force_merge_rejected", Key: "force_merge_rejected",
@ -589,33 +588,32 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
}) })
} }
//Get Thread Pool queue //Get Thread Pool queue
aggs:=map[string]interface{}{} aggs := map[string]interface{}{}
for _,metricItem:=range queueMetricItems{ for _, metricItem := range queueMetricItems {
aggs[metricItem.ID]=util.MapStr{ aggs[metricItem.ID] = util.MapStr{
"max":util.MapStr{ "max": util.MapStr{
"field": metricItem.Field, "field": metricItem.Field,
}, },
} }
if metricItem.Field2 != "" { if metricItem.Field2 != "" {
aggs[metricItem.ID + "_field2"]=util.MapStr{ aggs[metricItem.ID+"_field2"] = util.MapStr{
"max":util.MapStr{ "max": util.MapStr{
"field": metricItem.Field2, "field": metricItem.Field2,
}, },
} }
} }
if metricItem.IsDerivative{ if metricItem.IsDerivative {
aggs[metricItem.ID+"_deriv"]=util.MapStr{ aggs[metricItem.ID+"_deriv"] = util.MapStr{
"derivative":util.MapStr{ "derivative": util.MapStr{
"buckets_path": metricItem.ID, "buckets_path": metricItem.ID,
}, },
} }
if metricItem.Field2 != "" { if metricItem.Field2 != "" {
aggs[metricItem.ID + "_field2_deriv"]=util.MapStr{ aggs[metricItem.ID+"_field2_deriv"] = util.MapStr{
"derivative":util.MapStr{ "derivative": util.MapStr{
"buckets_path": metricItem.ID + "_field2", "buckets_path": metricItem.ID + "_field2",
}, },
} }
@ -628,8 +626,8 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
panic(err) panic(err)
} }
query["size"]=0 query["size"] = 0
query["aggs"]= util.MapStr{ query["aggs"] = util.MapStr{
"group_by_level": util.MapStr{ "group_by_level": util.MapStr{
"terms": util.MapStr{ "terms": util.MapStr{
"field": "metadata.labels.transport_address", "field": "metadata.labels.transport_address",
@ -637,11 +635,11 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string,
}, },
"aggs": util.MapStr{ "aggs": util.MapStr{
"dates": util.MapStr{ "dates": util.MapStr{
"date_histogram":util.MapStr{ "date_histogram": util.MapStr{
"field": "timestamp", "field": "timestamp",
intervalField: bucketSizeStr, intervalField: bucketSizeStr,
}, },
"aggs":aggs, "aggs": aggs,
}, },
}, },
}, },

37
modules/elastic/api/trace_template.go
View File

@ -38,10 +38,9 @@ import (
) )
func (h *APIHandler) HandleCrateTraceTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *APIHandler) HandleCrateTraceTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody := map[string] interface{}{ resBody := map[string]interface{}{}
}
targetClusterID := ps.ByName("id") targetClusterID := ps.ByName("id")
exists,client,err:=h.GetClusterClient(targetClusterID) exists, client, err := h.GetClusterClient(targetClusterID)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
@ -50,16 +49,14 @@ func (h *APIHandler) HandleCrateTraceTemplateAction(w http.ResponseWriter, req *
return return
} }
if !exists{ if !exists {
resBody["error"] = fmt.Sprintf("cluster [%s] not found",targetClusterID) resBody["error"] = fmt.Sprintf("cluster [%s] not found", targetClusterID)
log.Error(resBody["error"]) log.Error(resBody["error"])
h.WriteJSON(w, resBody, http.StatusNotFound) h.WriteJSON(w, resBody, http.StatusNotFound)
return return
} }
var traceReq = &elastic.TraceTemplate{ var traceReq = &elastic.TraceTemplate{}
}
err = h.DecodeJSON(req, traceReq) err = h.DecodeJSON(req, traceReq)
if err != nil { if err != nil {
@ -84,12 +81,11 @@ func (h *APIHandler) HandleCrateTraceTemplateAction(w http.ResponseWriter, req *
resBody["_id"] = insertRes.ID resBody["_id"] = insertRes.ID
resBody["result"] = insertRes.Result resBody["result"] = insertRes.Result
h.WriteJSON(w, resBody,http.StatusOK) h.WriteJSON(w, resBody, http.StatusOK)
} }
func (h *APIHandler) HandleSearchTraceTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *APIHandler) HandleSearchTraceTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody := map[string] interface{}{ resBody := map[string]interface{}{}
}
var ( var (
name = h.GetParameterOrDefault(req, "name", "") name = h.GetParameterOrDefault(req, "name", "")
queryDSL = `{"query":{"bool":{"must":[%s]}}, "size": %d, "from": %d}` queryDSL = `{"query":{"bool":{"must":[%s]}}, "size": %d, "from": %d}`
@ -99,7 +95,7 @@ func (h *APIHandler) HandleSearchTraceTemplateAction(w http.ResponseWriter, req
) )
targetClusterID := ps.ByName("id") targetClusterID := ps.ByName("id")
mustBuilder.WriteString(fmt.Sprintf(`{"term":{"cluster_id":{"value": "%s"}}}`, targetClusterID)) mustBuilder.WriteString(fmt.Sprintf(`{"term":{"cluster_id":{"value": "%s"}}}`, targetClusterID))
if name != ""{ if name != "" {
mustBuilder.WriteString(fmt.Sprintf(`,{"prefix":{"name": "%s"}}`, name)) mustBuilder.WriteString(fmt.Sprintf(`,{"prefix":{"name": "%s"}}`, name))
} }
size, _ := strconv.Atoi(strSize) size, _ := strconv.Atoi(strSize)
@ -126,8 +122,7 @@ func (h *APIHandler) HandleSearchTraceTemplateAction(w http.ResponseWriter, req
} }
func (h *APIHandler) HandleSaveTraceTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *APIHandler) HandleSaveTraceTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody := map[string]interface{}{ resBody := map[string]interface{}{}
}
reqParams := elastic.TraceTemplate{} reqParams := elastic.TraceTemplate{}
err := h.DecodeJSON(req, &reqParams) err := h.DecodeJSON(req, &reqParams)
@ -140,7 +135,7 @@ func (h *APIHandler) HandleSaveTraceTemplateAction(w http.ResponseWriter, req *h
reqParams.ID = ps.ByName("template_id") reqParams.ID = ps.ByName("template_id")
reqParams.Updated = time.Now() reqParams.Updated = time.Now()
esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
_, err = esClient.Index(orm.GetIndexName(reqParams),"", reqParams.ID, reqParams, "wait_for") _, err = esClient.Index(orm.GetIndexName(reqParams), "", reqParams.ID, reqParams, "wait_for")
if err != nil { if err != nil {
log.Error(err) log.Error(err)
resBody["error"] = err.Error() resBody["error"] = err.Error()
@ -152,11 +147,11 @@ func (h *APIHandler) HandleSaveTraceTemplateAction(w http.ResponseWriter, req *h
resBody["result"] = "updated" resBody["result"] = "updated"
resBody["_source"] = reqParams resBody["_source"] = reqParams
h.WriteJSON(w, resBody,http.StatusOK) h.WriteJSON(w, resBody, http.StatusOK)
} }
func (h *APIHandler) HandleGetTraceTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ func (h *APIHandler) HandleGetTraceTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody := map[string] interface{}{} resBody := map[string]interface{}{}
id := ps.ByName("template_id") id := ps.ByName("template_id")
indexName := orm.GetIndexName(elastic.TraceTemplate{}) indexName := orm.GetIndexName(elastic.TraceTemplate{})
@ -166,7 +161,7 @@ func (h *APIHandler) HandleGetTraceTemplateAction(w http.ResponseWriter, req *ht
resBody["error"] = err.Error() resBody["error"] = err.Error()
h.WriteJSON(w, resBody, http.StatusInternalServerError) h.WriteJSON(w, resBody, http.StatusInternalServerError)
} }
h.WriteJSON(w,getResponse, getResponse.StatusCode) h.WriteJSON(w, getResponse, getResponse.StatusCode)
} }
func (h *APIHandler) HandleDeleteTraceTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *APIHandler) HandleDeleteTraceTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
@ -177,9 +172,9 @@ func (h *APIHandler) HandleDeleteTraceTemplateAction(w http.ResponseWriter, req
if err != nil { if err != nil {
log.Error(err) log.Error(err)
resBody["error"] = err.Error() resBody["error"] = err.Error()
if delRes!=nil{ if delRes != nil {
h.WriteJSON(w, resBody, delRes.StatusCode) h.WriteJSON(w, resBody, delRes.StatusCode)
}else{ } else {
h.WriteJSON(w, resBody, http.StatusInternalServerError) h.WriteJSON(w, resBody, http.StatusInternalServerError)
} }
} }

4
modules/elastic/api/v1/cluster_overview.go
View File

@ -786,11 +786,11 @@ func (h *APIHandler) QueryQPS(query util.MapStr, bucketSizeInSeconds int) (map[s
if preIndexTotal > 0 { if preIndexTotal > 0 {
//if value of indexTotal is decreasing, drop the next value, //if value of indexTotal is decreasing, drop the next value,
//and we will drop current and next qps value //and we will drop current and next qps value
if indexTotalVal - preIndexTotal < 0 { if indexTotalVal-preIndexTotal < 0 {
dropNext = true dropNext = true
preIndexTotal = indexTotalVal preIndexTotal = indexTotalVal
continue continue
}else{ } else {
dropNext = false dropNext = false
} }
} }

179
modules/elastic/api/v1/index_metrics.go
View File

@ -77,16 +77,15 @@ const (
SegmentPointsMetricKey = "segment_points_memory" SegmentPointsMetricKey = "segment_points_memory"
VersionMapMetricKey = "segment_version_map" VersionMapMetricKey = "segment_version_map"
FixedBitSetMetricKey = "segment_fixed_bit_set" FixedBitSetMetricKey = "segment_fixed_bit_set"
) )
func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clusterID string, bucketSize int, min, max int64, indexName string, top int, metricKey string) (map[string]*common.MetricItem, error){ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clusterID string, bucketSize int, min, max int64, indexName string, top int, metricKey string) (map[string]*common.MetricItem, error) {
bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
var must = []util.MapStr{ var must = []util.MapStr{
{ {
"term":util.MapStr{ "term": util.MapStr{
"metadata.labels.cluster_id":util.MapStr{ "metadata.labels.cluster_id": util.MapStr{
"value": clusterID, "value": clusterID,
}, },
}, },
@ -116,11 +115,11 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
if !hasAllPrivilege && len(allowedIndices) == 0 { if !hasAllPrivilege && len(allowedIndices) == 0 {
return nil, nil return nil, nil
} }
if !hasAllPrivilege{ if !hasAllPrivilege {
namePattern := radix.Compile(allowedIndices...) namePattern := radix.Compile(allowedIndices...)
var filterNames []string var filterNames []string
for _, name := range indexNames { for _, name := range indexNames {
if namePattern.Match(name){ if namePattern.Match(name) {
filterNames = append(filterNames, name) filterNames = append(filterNames, name)
} }
} }
@ -131,7 +130,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
} }
top = len(indexNames) top = len(indexNames)
}else{ } else {
indexNames, err = h.getTopIndexName(req, clusterID, top, 15) indexNames, err = h.getTopIndexName(req, clusterID, top, 15)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
@ -146,8 +145,8 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
} }
query:=map[string]interface{}{} query := map[string]interface{}{}
query["query"]=util.MapStr{ query["query"] = util.MapStr{
"bool": util.MapStr{ "bool": util.MapStr{
"must": must, "must": must,
"must_not": []util.MapStr{ "must_not": []util.MapStr{
@ -295,7 +294,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case FlushTimesMetricKey: case FlushTimesMetricKey:
//flush 次数 //flush 次数
flushTimesMetric := newMetricItem(FlushTimesMetricKey, 6, OperationGroupKey) flushTimesMetric := newMetricItem(FlushTimesMetricKey, 6, OperationGroupKey)
flushTimesMetric.AddAxi("flush times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) flushTimesMetric.AddAxi("flush times", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "flush_times", Key: "flush_times",
Field: "payload.elasticsearch.index_stats.total.flush.total", Field: "payload.elasticsearch.index_stats.total.flush.total",
@ -308,7 +307,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case IndexingRateMetricKey: case IndexingRateMetricKey:
//写入速率 //写入速率
indexingRateMetric := newMetricItem(IndexingRateMetricKey, 1, OperationGroupKey) indexingRateMetric := newMetricItem(IndexingRateMetricKey, 1, OperationGroupKey)
indexingRateMetric.AddAxi("Indexing rate","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) indexingRateMetric.AddAxi("Indexing rate", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "indexing_rate", Key: "indexing_rate",
Field: "payload.elasticsearch.index_stats.primaries.indexing.index_total", Field: "payload.elasticsearch.index_stats.primaries.indexing.index_total",
@ -320,7 +319,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case IndexingBytesMetricKey: case IndexingBytesMetricKey:
indexingBytesMetric := newMetricItem(IndexingBytesMetricKey, 2, OperationGroupKey) indexingBytesMetric := newMetricItem(IndexingBytesMetricKey, 2, OperationGroupKey)
indexingBytesMetric.AddAxi("Indexing bytes","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) indexingBytesMetric.AddAxi("Indexing bytes", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "indexing_bytes", Key: "indexing_bytes",
Field: "payload.elasticsearch.index_stats.primaries.store.size_in_bytes", Field: "payload.elasticsearch.index_stats.primaries.store.size_in_bytes",
@ -333,13 +332,13 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case IndexingLatencyMetricKey: case IndexingLatencyMetricKey:
//写入时延 //写入时延
indexingLatencyMetric := newMetricItem(IndexingLatencyMetricKey, 1, LatencyGroupKey) indexingLatencyMetric := newMetricItem(IndexingLatencyMetricKey, 1, LatencyGroupKey)
indexingLatencyMetric.AddAxi("Indexing latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) indexingLatencyMetric.AddAxi("Indexing latency", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "indexing_latency", Key: "indexing_latency",
Field: "payload.elasticsearch.index_stats.primaries.indexing.index_time_in_millis", Field: "payload.elasticsearch.index_stats.primaries.indexing.index_time_in_millis",
Field2: "payload.elasticsearch.index_stats.primaries.indexing.index_total", Field2: "payload.elasticsearch.index_stats.primaries.indexing.index_total",
Calc: func(value, value2 float64) float64 { Calc: func(value, value2 float64) float64 {
return value/value2 return value / value2
}, },
ID: util.GetUUID(), ID: util.GetUUID(),
IsDerivative: true, IsDerivative: true,
@ -350,13 +349,13 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case QueryLatencyMetricKey: case QueryLatencyMetricKey:
//查询时延 //查询时延
queryLatencyMetric := newMetricItem(QueryLatencyMetricKey, 2, LatencyGroupKey) queryLatencyMetric := newMetricItem(QueryLatencyMetricKey, 2, LatencyGroupKey)
queryLatencyMetric.AddAxi("Query latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) queryLatencyMetric.AddAxi("Query latency", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "query_latency", Key: "query_latency",
Field: "payload.elasticsearch.index_stats.total.search.query_time_in_millis", Field: "payload.elasticsearch.index_stats.total.search.query_time_in_millis",
Field2: "payload.elasticsearch.index_stats.total.search.query_total", Field2: "payload.elasticsearch.index_stats.total.search.query_total",
Calc: func(value, value2 float64) float64 { Calc: func(value, value2 float64) float64 {
return value/value2 return value / value2
}, },
ID: util.GetUUID(), ID: util.GetUUID(),
IsDerivative: true, IsDerivative: true,
@ -367,13 +366,13 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case FetchLatencyMetricKey: case FetchLatencyMetricKey:
//fetch时延 //fetch时延
fetchLatencyMetric := newMetricItem(FetchLatencyMetricKey, 3, LatencyGroupKey) fetchLatencyMetric := newMetricItem(FetchLatencyMetricKey, 3, LatencyGroupKey)
fetchLatencyMetric.AddAxi("Fetch latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) fetchLatencyMetric.AddAxi("Fetch latency", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "fetch_latency", Key: "fetch_latency",
Field: "payload.elasticsearch.index_stats.total.search.fetch_time_in_millis", Field: "payload.elasticsearch.index_stats.total.search.fetch_time_in_millis",
Field2: "payload.elasticsearch.index_stats.total.search.fetch_total", Field2: "payload.elasticsearch.index_stats.total.search.fetch_total",
Calc: func(value, value2 float64) float64 { Calc: func(value, value2 float64) float64 {
return value/value2 return value / value2
}, },
ID: util.GetUUID(), ID: util.GetUUID(),
IsDerivative: true, IsDerivative: true,
@ -384,13 +383,13 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case MergeLatencyMetricKey: case MergeLatencyMetricKey:
//merge时延 //merge时延
mergeLatencyMetric := newMetricItem(MergeLatencyMetricKey, 7, LatencyGroupKey) mergeLatencyMetric := newMetricItem(MergeLatencyMetricKey, 7, LatencyGroupKey)
mergeLatencyMetric.AddAxi("Merge latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) mergeLatencyMetric.AddAxi("Merge latency", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "merge_latency", Key: "merge_latency",
Field: "payload.elasticsearch.index_stats.total.merges.total_time_in_millis", Field: "payload.elasticsearch.index_stats.total.merges.total_time_in_millis",
Field2: "payload.elasticsearch.index_stats.total.merges.total", Field2: "payload.elasticsearch.index_stats.total.merges.total",
Calc: func(value, value2 float64) float64 { Calc: func(value, value2 float64) float64 {
return value/value2 return value / value2
}, },
ID: util.GetUUID(), ID: util.GetUUID(),
IsDerivative: true, IsDerivative: true,
@ -402,13 +401,13 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
//refresh时延 //refresh时延
refreshLatencyMetric := newMetricItem(RefreshLatencyMetricKey, 5, LatencyGroupKey) refreshLatencyMetric := newMetricItem(RefreshLatencyMetricKey, 5, LatencyGroupKey)
refreshLatencyMetric.AddAxi("Refresh latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) refreshLatencyMetric.AddAxi("Refresh latency", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "refresh_latency", Key: "refresh_latency",
Field: "payload.elasticsearch.index_stats.total.refresh.total_time_in_millis", Field: "payload.elasticsearch.index_stats.total.refresh.total_time_in_millis",
Field2: "payload.elasticsearch.index_stats.total.refresh.total", Field2: "payload.elasticsearch.index_stats.total.refresh.total",
Calc: func(value, value2 float64) float64 { Calc: func(value, value2 float64) float64 {
return value/value2 return value / value2
}, },
ID: util.GetUUID(), ID: util.GetUUID(),
IsDerivative: true, IsDerivative: true,
@ -419,13 +418,13 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case ScrollLatencyMetricKey: case ScrollLatencyMetricKey:
//scroll时延 //scroll时延
scrollLatencyMetric := newMetricItem(ScrollLatencyMetricKey, 4, LatencyGroupKey) scrollLatencyMetric := newMetricItem(ScrollLatencyMetricKey, 4, LatencyGroupKey)
scrollLatencyMetric.AddAxi("Scroll Latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) scrollLatencyMetric.AddAxi("Scroll Latency", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "scroll_latency", Key: "scroll_latency",
Field: "payload.elasticsearch.index_stats.total.search.scroll_time_in_millis", Field: "payload.elasticsearch.index_stats.total.search.scroll_time_in_millis",
Field2: "payload.elasticsearch.index_stats.total.search.scroll_total", Field2: "payload.elasticsearch.index_stats.total.search.scroll_total",
Calc: func(value, value2 float64) float64 { Calc: func(value, value2 float64) float64 {
return value/value2 return value / value2
}, },
ID: util.GetUUID(), ID: util.GetUUID(),
IsDerivative: true, IsDerivative: true,
@ -436,13 +435,13 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case FlushLatencyMetricKey: case FlushLatencyMetricKey:
//flush 时延 //flush 时延
flushLatencyMetric := newMetricItem(FlushLatencyMetricKey, 6, LatencyGroupKey) flushLatencyMetric := newMetricItem(FlushLatencyMetricKey, 6, LatencyGroupKey)
flushLatencyMetric.AddAxi("Flush latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) flushLatencyMetric.AddAxi("Flush latency", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "flush_latency", Key: "flush_latency",
Field: "payload.elasticsearch.index_stats.total.flush.total_time_in_millis", Field: "payload.elasticsearch.index_stats.total.flush.total_time_in_millis",
Field2: "payload.elasticsearch.index_stats.total.flush.total", Field2: "payload.elasticsearch.index_stats.total.flush.total",
Calc: func(value, value2 float64) float64 { Calc: func(value, value2 float64) float64 {
return value/value2 return value / value2
}, },
ID: util.GetUUID(), ID: util.GetUUID(),
IsDerivative: true, IsDerivative: true,
@ -453,7 +452,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case QueryCacheMetricKey: case QueryCacheMetricKey:
//queryCache //queryCache
queryCacheMetric := newMetricItem(QueryCacheMetricKey, 1, CacheGroupKey) queryCacheMetric := newMetricItem(QueryCacheMetricKey, 1, CacheGroupKey)
queryCacheMetric.AddAxi("Query cache","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) queryCacheMetric.AddAxi("Query cache", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "query_cache", Key: "query_cache",
Field: "payload.elasticsearch.index_stats.total.query_cache.memory_size_in_bytes", Field: "payload.elasticsearch.index_stats.total.query_cache.memory_size_in_bytes",
@ -466,7 +465,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case RequestCacheMetricKey: case RequestCacheMetricKey:
//requestCache //requestCache
requestCacheMetric := newMetricItem(RequestCacheMetricKey, 2, CacheGroupKey) requestCacheMetric := newMetricItem(RequestCacheMetricKey, 2, CacheGroupKey)
requestCacheMetric.AddAxi("request cache","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) requestCacheMetric.AddAxi("request cache", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "request_cache", Key: "request_cache",
Field: "payload.elasticsearch.index_stats.total.request_cache.memory_size_in_bytes", Field: "payload.elasticsearch.index_stats.total.request_cache.memory_size_in_bytes",
@ -478,9 +477,9 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case RequestCacheHitMetricKey: case RequestCacheHitMetricKey:
// Request Cache Hit // Request Cache Hit
requestCacheHitMetric:=newMetricItem(RequestCacheHitMetricKey, 6, CacheGroupKey) requestCacheHitMetric := newMetricItem(RequestCacheHitMetricKey, 6, CacheGroupKey)
requestCacheHitMetric.AddAxi("request cache hit","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) requestCacheHitMetric.AddAxi("request cache hit", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
indexMetricItems=append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "request_cache_hit", Key: "request_cache_hit",
Field: "payload.elasticsearch.index_stats.total.request_cache.hit_count", Field: "payload.elasticsearch.index_stats.total.request_cache.hit_count",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -491,9 +490,9 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case RequestCacheMissMetricKey: case RequestCacheMissMetricKey:
// Request Cache Miss // Request Cache Miss
requestCacheMissMetric:=newMetricItem(RequestCacheMissMetricKey, 8, CacheGroupKey) requestCacheMissMetric := newMetricItem(RequestCacheMissMetricKey, 8, CacheGroupKey)
requestCacheMissMetric.AddAxi("request cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) requestCacheMissMetric.AddAxi("request cache miss", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
indexMetricItems=append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "request_cache_miss", Key: "request_cache_miss",
Field: "payload.elasticsearch.index_stats.total.request_cache.miss_count", Field: "payload.elasticsearch.index_stats.total.request_cache.miss_count",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -504,9 +503,9 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case QueryCacheCountMetricKey: case QueryCacheCountMetricKey:
// Query Cache Count // Query Cache Count
queryCacheCountMetric:=newMetricItem(QueryCacheCountMetricKey, 4, CacheGroupKey) queryCacheCountMetric := newMetricItem(QueryCacheCountMetricKey, 4, CacheGroupKey)
queryCacheCountMetric.AddAxi("query cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) queryCacheCountMetric.AddAxi("query cache miss", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
indexMetricItems=append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "query_cache_count", Key: "query_cache_count",
Field: "payload.elasticsearch.index_stats.total.query_cache.cache_count", Field: "payload.elasticsearch.index_stats.total.query_cache.cache_count",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -517,9 +516,9 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case QueryCacheHitMetricKey: case QueryCacheHitMetricKey:
// Query Cache Miss // Query Cache Miss
queryCacheHitMetric:=newMetricItem(QueryCacheHitMetricKey, 5, CacheGroupKey) queryCacheHitMetric := newMetricItem(QueryCacheHitMetricKey, 5, CacheGroupKey)
queryCacheHitMetric.AddAxi("query cache hit","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) queryCacheHitMetric.AddAxi("query cache hit", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
indexMetricItems=append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "query_cache_hit", Key: "query_cache_hit",
Field: "payload.elasticsearch.index_stats.total.query_cache.hit_count", Field: "payload.elasticsearch.index_stats.total.query_cache.hit_count",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -530,9 +529,9 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case QueryCacheMissMetricKey: case QueryCacheMissMetricKey:
// Query Cache Miss // Query Cache Miss
queryCacheMissMetric:=newMetricItem(QueryCacheMissMetricKey, 7, CacheGroupKey) queryCacheMissMetric := newMetricItem(QueryCacheMissMetricKey, 7, CacheGroupKey)
queryCacheMissMetric.AddAxi("query cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) queryCacheMissMetric.AddAxi("query cache miss", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
indexMetricItems=append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "query_cache_miss", Key: "query_cache_miss",
Field: "payload.elasticsearch.index_stats.total.query_cache.miss_count", Field: "payload.elasticsearch.index_stats.total.query_cache.miss_count",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -543,9 +542,9 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case FielddataCacheMetricKey: case FielddataCacheMetricKey:
// Fielddata内存占用大小 // Fielddata内存占用大小
fieldDataCacheMetric:=newMetricItem(FielddataCacheMetricKey, 3, CacheGroupKey) fieldDataCacheMetric := newMetricItem(FielddataCacheMetricKey, 3, CacheGroupKey)
fieldDataCacheMetric.AddAxi("FieldData Cache","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) fieldDataCacheMetric.AddAxi("FieldData Cache", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, true)
indexMetricItems=append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "fielddata_cache", Key: "fielddata_cache",
Field: "payload.elasticsearch.index_stats.total.fielddata.memory_size_in_bytes", Field: "payload.elasticsearch.index_stats.total.fielddata.memory_size_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -557,7 +556,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case SegmentMemoryMetricKey: case SegmentMemoryMetricKey:
//segment memory //segment memory
segmentMemoryMetric := newMetricItem(SegmentMemoryMetricKey, 13, MemoryGroupKey) segmentMemoryMetric := newMetricItem(SegmentMemoryMetricKey, 13, MemoryGroupKey)
segmentMemoryMetric.AddAxi("Segment memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) segmentMemoryMetric.AddAxi("Segment memory", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "segment_memory", Key: "segment_memory",
Field: "payload.elasticsearch.index_stats.total.segments.memory_in_bytes", Field: "payload.elasticsearch.index_stats.total.segments.memory_in_bytes",
@ -570,7 +569,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case SegmentDocValuesMemoryMetricKey: case SegmentDocValuesMemoryMetricKey:
//segment doc values memory //segment doc values memory
docValuesMemoryMetric := newMetricItem(SegmentDocValuesMemoryMetricKey, 13, MemoryGroupKey) docValuesMemoryMetric := newMetricItem(SegmentDocValuesMemoryMetricKey, 13, MemoryGroupKey)
docValuesMemoryMetric.AddAxi("Segment Doc values Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) docValuesMemoryMetric.AddAxi("Segment Doc values Memory", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "segment_doc_values_memory", Key: "segment_doc_values_memory",
Field: "payload.elasticsearch.index_stats.total.segments.doc_values_memory_in_bytes", Field: "payload.elasticsearch.index_stats.total.segments.doc_values_memory_in_bytes",
@ -583,7 +582,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case SegmentTermsMemoryMetricKey: case SegmentTermsMemoryMetricKey:
//segment terms memory //segment terms memory
termsMemoryMetric := newMetricItem(SegmentTermsMemoryMetricKey, 13, MemoryGroupKey) termsMemoryMetric := newMetricItem(SegmentTermsMemoryMetricKey, 13, MemoryGroupKey)
termsMemoryMetric.AddAxi("Segment Terms Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) termsMemoryMetric.AddAxi("Segment Terms Memory", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "segment_terms_memory", Key: "segment_terms_memory",
Field: "payload.elasticsearch.index_stats.total.segments.terms_memory_in_bytes", Field: "payload.elasticsearch.index_stats.total.segments.terms_memory_in_bytes",
@ -596,7 +595,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
case SegmentFieldsMemoryMetricKey: case SegmentFieldsMemoryMetricKey:
//segment fields memory //segment fields memory
fieldsMemoryMetric := newMetricItem(SegmentFieldsMemoryMetricKey, 13, MemoryGroupKey) fieldsMemoryMetric := newMetricItem(SegmentFieldsMemoryMetricKey, 13, MemoryGroupKey)
fieldsMemoryMetric.AddAxi("Segment Fields Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) fieldsMemoryMetric.AddAxi("Segment Fields Memory", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "segment_fields_memory", Key: "segment_fields_memory",
Field: "payload.elasticsearch.index_stats.total.segments.stored_fields_memory_in_bytes", Field: "payload.elasticsearch.index_stats.total.segments.stored_fields_memory_in_bytes",
@ -608,9 +607,9 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case SegmentIndexWriterMemoryMetricKey: case SegmentIndexWriterMemoryMetricKey:
// segment index writer memory // segment index writer memory
segmentIndexWriterMemoryMetric:=newMetricItem(SegmentIndexWriterMemoryMetricKey, 16, MemoryGroupKey) segmentIndexWriterMemoryMetric := newMetricItem(SegmentIndexWriterMemoryMetricKey, 16, MemoryGroupKey)
segmentIndexWriterMemoryMetric.AddAxi("segment doc values memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) segmentIndexWriterMemoryMetric.AddAxi("segment doc values memory", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, true)
indexMetricItems=append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "segment_index_writer_memory", Key: "segment_index_writer_memory",
Field: "payload.elasticsearch.index_stats.total.segments.index_writer_memory_in_bytes", Field: "payload.elasticsearch.index_stats.total.segments.index_writer_memory_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -621,9 +620,9 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case SegmentTermVectorsMemoryMetricKey: case SegmentTermVectorsMemoryMetricKey:
// segment term vectors memory // segment term vectors memory
segmentTermVectorsMemoryMetric:=newMetricItem(SegmentTermVectorsMemoryMetricKey, 16, MemoryGroupKey) segmentTermVectorsMemoryMetric := newMetricItem(SegmentTermVectorsMemoryMetricKey, 16, MemoryGroupKey)
segmentTermVectorsMemoryMetric.AddAxi("segment term vectors memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) segmentTermVectorsMemoryMetric.AddAxi("segment term vectors memory", "group1", common.PositionLeft, "bytes", "0,0", "0,0.[00]", 5, true)
indexMetricItems=append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: "segment_term_vectors_memory", Key: "segment_term_vectors_memory",
Field: "payload.elasticsearch.index_stats.total.segments.term_vectors_memory_in_bytes", Field: "payload.elasticsearch.index_stats.total.segments.term_vectors_memory_in_bytes",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -634,7 +633,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case SegmentNormsMetricKey: case SegmentNormsMetricKey:
segmentNormsMetric := newMetricItem(SegmentNormsMetricKey, 17, MemoryGroupKey) segmentNormsMetric := newMetricItem(SegmentNormsMetricKey, 17, MemoryGroupKey)
segmentNormsMetric.AddAxi("Segment norms memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) segmentNormsMetric.AddAxi("Segment norms memory", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: SegmentNormsMetricKey, Key: SegmentNormsMetricKey,
Field: "payload.elasticsearch.index_stats.total.segments.norms_memory_in_bytes", Field: "payload.elasticsearch.index_stats.total.segments.norms_memory_in_bytes",
@ -646,7 +645,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case SegmentPointsMetricKey: case SegmentPointsMetricKey:
segmentPointsMetric := newMetricItem(SegmentPointsMetricKey, 18, MemoryGroupKey) segmentPointsMetric := newMetricItem(SegmentPointsMetricKey, 18, MemoryGroupKey)
segmentPointsMetric.AddAxi("Segment points memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) segmentPointsMetric.AddAxi("Segment points memory", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: SegmentPointsMetricKey, Key: SegmentPointsMetricKey,
Field: "payload.elasticsearch.index_stats.total.segments.points_memory_in_bytes", Field: "payload.elasticsearch.index_stats.total.segments.points_memory_in_bytes",
@ -658,7 +657,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case VersionMapMetricKey: case VersionMapMetricKey:
segmentVersionMapMetric := newMetricItem(VersionMapMetricKey, 18, MemoryGroupKey) segmentVersionMapMetric := newMetricItem(VersionMapMetricKey, 18, MemoryGroupKey)
segmentVersionMapMetric.AddAxi("Segment version map memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) segmentVersionMapMetric.AddAxi("Segment version map memory", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: VersionMapMetricKey, Key: VersionMapMetricKey,
Field: "payload.elasticsearch.index_stats.total.segments.version_map_memory_in_bytes", Field: "payload.elasticsearch.index_stats.total.segments.version_map_memory_in_bytes",
@ -670,7 +669,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
case FixedBitSetMetricKey: case FixedBitSetMetricKey:
segmentFixedBitSetMetric := newMetricItem(FixedBitSetMetricKey, 18, MemoryGroupKey) segmentFixedBitSetMetric := newMetricItem(FixedBitSetMetricKey, 18, MemoryGroupKey)
segmentFixedBitSetMetric.AddAxi("Segment fixed bit set memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) segmentFixedBitSetMetric.AddAxi("Segment fixed bit set memory", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
indexMetricItems = append(indexMetricItems, GroupMetricItem{ indexMetricItems = append(indexMetricItems, GroupMetricItem{
Key: FixedBitSetMetricKey, Key: FixedBitSetMetricKey,
Field: "payload.elasticsearch.index_stats.total.segments.fixed_bit_set_memory_in_bytes", Field: "payload.elasticsearch.index_stats.total.segments.fixed_bit_set_memory_in_bytes",
@ -682,33 +681,32 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}) })
} }
aggs := map[string]interface{}{}
aggs:=map[string]interface{}{} for _, metricItem := range indexMetricItems {
aggs[metricItem.ID] = util.MapStr{
for _,metricItem:=range indexMetricItems { "max": util.MapStr{
aggs[metricItem.ID]=util.MapStr{
"max":util.MapStr{
"field": metricItem.Field, "field": metricItem.Field,
}, },
} }
if metricItem.Field2 != ""{ if metricItem.Field2 != "" {
aggs[metricItem.ID + "_field2"]=util.MapStr{ aggs[metricItem.ID+"_field2"] = util.MapStr{
"max":util.MapStr{ "max": util.MapStr{
"field": metricItem.Field2, "field": metricItem.Field2,
}, },
} }
} }
if metricItem.IsDerivative{ if metricItem.IsDerivative {
aggs[metricItem.ID+"_deriv"]=util.MapStr{ aggs[metricItem.ID+"_deriv"] = util.MapStr{
"derivative":util.MapStr{ "derivative": util.MapStr{
"buckets_path": metricItem.ID, "buckets_path": metricItem.ID,
}, },
} }
if metricItem.Field2 != "" { if metricItem.Field2 != "" {
aggs[metricItem.ID + "_deriv_field2"]=util.MapStr{ aggs[metricItem.ID+"_deriv_field2"] = util.MapStr{
"derivative":util.MapStr{ "derivative": util.MapStr{
"buckets_path": metricItem.ID + "_field2", "buckets_path": metricItem.ID + "_field2",
}, },
} }
@ -720,8 +718,8 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
return nil, err return nil, err
} }
query["size"]=0 query["size"] = 0
query["aggs"]= util.MapStr{ query["aggs"] = util.MapStr{
"group_by_level": util.MapStr{ "group_by_level": util.MapStr{
"terms": util.MapStr{ "terms": util.MapStr{
"field": "metadata.labels.index_name", "field": "metadata.labels.index_name",
@ -732,11 +730,11 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
}, },
"aggs": util.MapStr{ "aggs": util.MapStr{
"dates": util.MapStr{ "dates": util.MapStr{
"date_histogram":util.MapStr{ "date_histogram": util.MapStr{
"field": "timestamp", "field": "timestamp",
intervalField: bucketSizeStr, intervalField: bucketSizeStr,
}, },
"aggs":aggs, "aggs": aggs,
}, },
"max_store": util.MapStr{ "max_store": util.MapStr{
"max": util.MapStr{ "max": util.MapStr{
@ -750,7 +748,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu
} }
func (h *APIHandler) getTopIndexName(req *http.Request, clusterID string, top int, lastMinutes int) ([]string, error){ func (h *APIHandler) getTopIndexName(req *http.Request, clusterID string, top int, lastMinutes int) ([]string, error) {
ver := h.Client().GetVersion() ver := h.Client().GetVersion()
cr, _ := util.VersionCompare(ver.Number, "6.1") cr, _ := util.VersionCompare(ver.Number, "6.1")
if (ver.Distribution == "" || ver.Distribution == elastic.Elasticsearch) && cr == -1 { if (ver.Distribution == "" || ver.Distribution == elastic.Elasticsearch) && cr == -1 {
@ -758,8 +756,8 @@ func (h *APIHandler) getTopIndexName(req *http.Request, clusterID string, top in
} }
var ( var (
now = time.Now() now = time.Now()
max = now.UnixNano()/1e6 max = now.UnixNano() / 1e6
min = now.Add(-time.Duration(lastMinutes) * time.Minute).UnixNano()/1e6 min = now.Add(-time.Duration(lastMinutes)*time.Minute).UnixNano() / 1e6
) )
var must = []util.MapStr{ var must = []util.MapStr{
{ {
@ -909,20 +907,20 @@ func (h *APIHandler) getTopIndexName(req *http.Request, clusterID string, top in
}, },
}, },
} }
response,err:=elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(),util.MustToJSONBytes(query)) response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query))
if err!=nil{ if err != nil {
log.Error(err) log.Error(err)
return nil, err return nil, err
} }
var maxQpsKVS = map[string] float64{} var maxQpsKVS = map[string]float64{}
for _, agg := range response.Aggregations { for _, agg := range response.Aggregations {
for _, bk := range agg.Buckets { for _, bk := range agg.Buckets {
key := bk["key"].(string) key := bk["key"].(string)
if maxQps, ok := bk["max_qps"].(map[string]interface{}); ok { if maxQps, ok := bk["max_qps"].(map[string]interface{}); ok {
val := maxQps["value"].(float64) val := maxQps["value"].(float64)
if _, ok = maxQpsKVS[key] ; ok { if _, ok = maxQpsKVS[key]; ok {
maxQpsKVS[key] = maxQpsKVS[key] + val maxQpsKVS[key] = maxQpsKVS[key] + val
}else{ } else {
maxQpsKVS[key] = val maxQpsKVS[key] = val
} }
} }
@ -943,7 +941,7 @@ func (h *APIHandler) getTopIndexName(req *http.Request, clusterID string, top in
length = len(qpsValues) length = len(qpsValues)
} }
indexNames := []string{} indexNames := []string{}
for i := 0; i <length; i++ { for i := 0; i < length; i++ {
indexNames = append(indexNames, qpsValues[i].Key) indexNames = append(indexNames, qpsValues[i].Key)
} }
return indexNames, nil return indexNames, nil
@ -954,12 +952,13 @@ type TopTerm struct {
Value float64 Value float64
} }
type TopTermOrder []TopTerm type TopTermOrder []TopTerm
func (t TopTermOrder) Len() int{
func (t TopTermOrder) Len() int {
return len(t) return len(t)
} }
func (t TopTermOrder) Less(i, j int) bool{ func (t TopTermOrder) Less(i, j int) bool {
return t[i].Value > t[j].Value //desc return t[i].Value > t[j].Value //desc
} }
func (t TopTermOrder) Swap(i, j int){ func (t TopTermOrder) Swap(i, j int) {
t[i], t[j] = t[j], t[i] t[i], t[j] = t[j], t[i]
} }

106
modules/elastic/api/v1/index_overview.go
View File

@ -97,7 +97,7 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, ctx context.Context,
if sinfo, ok := shardInfo.([]interface{}); ok { if sinfo, ok := shardInfo.([]interface{}); ok {
unassignedCount := 0 unassignedCount := 0
for _, item := range sinfo { for _, item := range sinfo {
if itemMap, ok := item.(map[string]interface{}); ok{ if itemMap, ok := item.(map[string]interface{}); ok {
if itemMap["state"] == "UNASSIGNED" { if itemMap["state"] == "UNASSIGNED" {
unassignedCount++ unassignedCount++
} }
@ -121,7 +121,7 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, ctx context.Context,
return return
} }
firstClusterID, firstIndexName = parts[0], parts[1] firstClusterID, firstIndexName = parts[0], parts[1]
}else{ } else {
h.WriteError(w, fmt.Sprintf("invalid index_id: %v", indexID), http.StatusInternalServerError) h.WriteError(w, fmt.Sprintf("invalid index_id: %v", indexID), http.StatusInternalServerError)
return return
} }
@ -137,10 +137,10 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, ctx context.Context,
} }
var metricLen = 15 var metricLen = 15
// 索引速率 // 索引速率
indexMetric:=newMetricItem("indexing", 1, OperationGroupKey) indexMetric := newMetricItem("indexing", 1, OperationGroupKey)
indexMetric.AddAxi("indexing rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) indexMetric.AddAxi("indexing rate", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems := []GroupMetricItem{} nodeMetricItems := []GroupMetricItem{}
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "indexing", Key: "indexing",
Field: "payload.elasticsearch.index_stats.primaries.indexing.index_total", Field: "payload.elasticsearch.index_stats.primaries.indexing.index_total",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -149,9 +149,9 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, ctx context.Context,
FormatType: "num", FormatType: "num",
Units: "Indexing/s", Units: "Indexing/s",
}) })
queryMetric:=newMetricItem("search", 2, OperationGroupKey) queryMetric := newMetricItem("search", 2, OperationGroupKey)
queryMetric.AddAxi("query rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) queryMetric.AddAxi("query rate", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "search", Key: "search",
Field: "payload.elasticsearch.index_stats.total.search.query_total", Field: "payload.elasticsearch.index_stats.total.search.query_total",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -161,9 +161,9 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, ctx context.Context,
Units: "Search/s", Units: "Search/s",
}) })
aggs:=map[string]interface{}{} aggs := map[string]interface{}{}
query :=map[string]interface{}{} query := map[string]interface{}{}
query["query"]=util.MapStr{ query["query"] = util.MapStr{
"bool": util.MapStr{ "bool": util.MapStr{
"must": []util.MapStr{ "must": []util.MapStr{
{ {
@ -190,7 +190,7 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, ctx context.Context,
{ {
"range": util.MapStr{ "range": util.MapStr{
"timestamp": util.MapStr{ "timestamp": util.MapStr{
"gte": fmt.Sprintf("now-%ds", metricLen * bucketSize), "gte": fmt.Sprintf("now-%ds", metricLen*bucketSize),
}, },
}, },
}, },
@ -198,15 +198,15 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, ctx context.Context,
}, },
} }
for _,metricItem:=range nodeMetricItems{ for _, metricItem := range nodeMetricItems {
aggs[metricItem.ID]=util.MapStr{ aggs[metricItem.ID] = util.MapStr{
"max":util.MapStr{ "max": util.MapStr{
"field": metricItem.Field, "field": metricItem.Field,
}, },
} }
if metricItem.IsDerivative{ if metricItem.IsDerivative {
aggs[metricItem.ID+"_deriv"]=util.MapStr{ aggs[metricItem.ID+"_deriv"] = util.MapStr{
"derivative":util.MapStr{ "derivative": util.MapStr{
"buckets_path": metricItem.ID, "buckets_path": metricItem.ID,
}, },
} }
@ -218,8 +218,8 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, ctx context.Context,
if err != nil { if err != nil {
panic(err) panic(err)
} }
query["size"]=0 query["size"] = 0
query["aggs"]= util.MapStr{ query["aggs"] = util.MapStr{
"group_by_level": util.MapStr{ "group_by_level": util.MapStr{
"terms": util.MapStr{ "terms": util.MapStr{
"field": "metadata.labels.index_id", "field": "metadata.labels.index_id",
@ -227,11 +227,11 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, ctx context.Context,
}, },
"aggs": util.MapStr{ "aggs": util.MapStr{
"dates": util.MapStr{ "dates": util.MapStr{
"date_histogram":util.MapStr{ "date_histogram": util.MapStr{
"field": "timestamp", "field": "timestamp",
intervalField: bucketSizeStr, intervalField: bucketSizeStr,
}, },
"aggs":aggs, "aggs": aggs,
}, },
}, },
}, },
@ -245,9 +245,8 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, ctx context.Context,
indexMetrics := map[string]util.MapStr{} indexMetrics := map[string]util.MapStr{}
for key, item := range metrics { for key, item := range metrics {
for _, line := range item.Lines { for _, line := range item.Lines {
if _, ok := indexMetrics[line.Metric.Label]; !ok{ if _, ok := indexMetrics[line.Metric.Label]; !ok {
indexMetrics[line.Metric.Label] = util.MapStr{ indexMetrics[line.Metric.Label] = util.MapStr{}
}
} }
indexMetrics[line.Metric.Label][key] = line.Data indexMetrics[line.Metric.Label][key] = line.Data
} }
@ -296,7 +295,7 @@ func (h *APIHandler) GetIndexInfo(w http.ResponseWriter, req *http.Request, ps h
return return
} }
if len(parts) < 2 { if len(parts) < 2 {
h.WriteError(w, "invalid index id: "+ indexID, http.StatusInternalServerError) h.WriteError(w, "invalid index id: "+indexID, http.StatusInternalServerError)
return return
} }
@ -340,7 +339,7 @@ func (h *APIHandler) GetIndexInfo(w http.ResponseWriter, req *http.Request, ps h
summary["aliases"] = aliases summary["aliases"] = aliases
summary["timestamp"] = hit["timestamp"] summary["timestamp"] = hit["timestamp"]
summary["index_info"] = util.MapStr{ summary["index_info"] = util.MapStr{
"health":health, "health": health,
"status": state, "status": state,
} }
} }
@ -361,11 +360,11 @@ func (h *APIHandler) GetIndexInfo(w http.ResponseWriter, req *http.Request, ps h
if tm, ok := result["timestamp"].(string); ok { if tm, ok := result["timestamp"].(string); ok {
issueTime, _ := time.Parse(time.RFC3339, tm) issueTime, _ := time.Parse(time.RFC3339, tm)
if time.Now().Sub(issueTime).Seconds() > 30 { if time.Now().Sub(issueTime).Seconds() > 30 {
health, _:= util.GetMapValueByKeys([]string{"metadata", "labels", "health_status"}, response.Hits.Hits[0].Source) health, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "health_status"}, response.Hits.Hits[0].Source)
infoM["health"] = health infoM["health"] = health
} }
} }
state, _:= util.GetMapValueByKeys([]string{"metadata", "labels", "state"}, response.Hits.Hits[0].Source) state, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "state"}, response.Hits.Hits[0].Source)
if state == "delete" { if state == "delete" {
infoM["status"] = "delete" infoM["status"] = "delete"
infoM["health"] = "N/A" infoM["health"] = "N/A"
@ -377,7 +376,7 @@ func (h *APIHandler) GetIndexInfo(w http.ResponseWriter, req *http.Request, ps h
if sinfo, ok := shardInfo.([]interface{}); ok { if sinfo, ok := shardInfo.([]interface{}); ok {
unassignedCount := 0 unassignedCount := 0
for _, item := range sinfo { for _, item := range sinfo {
if itemMap, ok := item.(map[string]interface{}); ok{ if itemMap, ok := item.(map[string]interface{}); ok {
if itemMap["state"] == "UNASSIGNED" { if itemMap["state"] == "UNASSIGNED" {
unassignedCount++ unassignedCount++
} }
@ -411,9 +410,9 @@ func (h *APIHandler) GetIndexShards(w http.ResponseWriter, req *http.Request, ps
q1.AddSort("timestamp", orm.DESC) q1.AddSort("timestamp", orm.DESC)
err, result := orm.Search(&event.Event{}, &q1) err, result := orm.Search(&event.Event{}, &q1)
if err != nil { if err != nil {
h.WriteJSON(w,util.MapStr{ h.WriteJSON(w, util.MapStr{
"error": err.Error(), "error": err.Error(),
}, http.StatusInternalServerError ) }, http.StatusInternalServerError)
return return
} }
var shardInfo interface{} = []interface{}{} var shardInfo interface{} = []interface{}{}
@ -512,7 +511,7 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ
log.Error(err) log.Error(err)
} }
metrics["index_health"] = healthMetric metrics["index_health"] = healthMetric
}else { } else {
switch metricKey { switch metricKey {
case IndexThroughputMetricKey: case IndexThroughputMetricKey:
metricItem := newMetricItem("index_throughput", 1, OperationGroupKey) metricItem := newMetricItem("index_throughput", 1, OperationGroupKey)
@ -582,7 +581,7 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ
minBucketSize, err := GetMetricMinBucketSize(clusterID, MetricTypeIndexStats) minBucketSize, err := GetMetricMinBucketSize(clusterID, MetricTypeIndexStats)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
}else{ } else {
metrics[metricKey].MinBucketSize = int64(minBucketSize) metrics[metricKey].MinBucketSize = int64(minBucketSize)
} }
} }
@ -591,8 +590,8 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ
h.WriteJSON(w, resBody, http.StatusOK) h.WriteJSON(w, resBody, http.StatusOK)
} }
func (h *APIHandler) GetIndexHealthMetric(ctx context.Context, id, indexName string, min, max int64, bucketSize int)(*common.MetricItem, error){ func (h *APIHandler) GetIndexHealthMetric(ctx context.Context, id, indexName string, min, max int64, bucketSize int) (*common.MetricItem, error) {
bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr)
if err != nil { if err != nil {
return nil, err return nil, err
@ -666,8 +665,8 @@ func (h *APIHandler) GetIndexHealthMetric(ctx context.Context, id, indexName str
return nil, err return nil, err
} }
metricItem:=newMetricItem("index_health", 1, "") metricItem := newMetricItem("index_health", 1, "")
metricItem.AddLine("health","Health","","group1","payload.elasticsearch.index_health.status","max",bucketSizeStr,"%","ratio","0.[00]","0.[00]",false,false) metricItem.AddLine("health", "Health", "", "group1", "payload.elasticsearch.index_health.status", "max", bucketSizeStr, "%", "ratio", "0.[00]", "0.[00]", false, false)
metricData := []interface{}{} metricData := []interface{}{}
if response.StatusCode == 200 { if response.StatusCode == 200 {
@ -683,8 +682,7 @@ func (h *APIHandler) GetIndexHealthMetric(ctx context.Context, id, indexName str
return metricItem, nil return metricItem, nil
} }
func (h *APIHandler) GetIndexStatusOfRecentDay(clusterID, indexName string) (map[string][]interface{}, error) {
func (h *APIHandler) GetIndexStatusOfRecentDay(clusterID, indexName string)(map[string][]interface{}, error){
q := orm.Query{ q := orm.Query{
WildcardIndex: true, WildcardIndex: true,
} }
@ -732,7 +730,7 @@ func (h *APIHandler) GetIndexStatusOfRecentDay(clusterID, indexName string)(map[
{ {
"from": "now-4d/d", "from": "now-4d/d",
"to": "now-3d/d", "to": "now-3d/d",
},{ }, {
"from": "now-3d/d", "from": "now-3d/d",
"to": "now-2d/d", "to": "now-2d/d",
}, { }, {
@ -824,9 +822,9 @@ func (h *APIHandler) GetIndexStatusOfRecentDay(clusterID, indexName string)(map[
} }
if _, ok = healthMap["red"]; ok { if _, ok = healthMap["red"]; ok {
status = "red" status = "red"
}else if _, ok = healthMap["yellow"]; ok { } else if _, ok = healthMap["yellow"]; ok {
status = "yellow" status = "yellow"
}else if _, ok = healthMap["green"]; ok { } else if _, ok = healthMap["green"]; ok {
status = "green" status = "green"
} }
key := fmt.Sprintf("%s:%s", clusterID, indexName) key := fmt.Sprintf("%s:%s", clusterID, indexName)
@ -838,7 +836,7 @@ func (h *APIHandler) GetIndexStatusOfRecentDay(clusterID, indexName string)(map[
} }
func (h *APIHandler) getIndexNodes(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *APIHandler) getIndexNodes(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody := map[string] interface{}{} resBody := map[string]interface{}{}
id := ps.ByName("id") id := ps.ByName("id")
indexName := ps.ByName("index") indexName := ps.ByName("index")
if !h.IsIndexAllowed(req, id, indexName) { if !h.IsIndexAllowed(req, id, indexName) {
@ -847,7 +845,7 @@ func (h *APIHandler) getIndexNodes(w http.ResponseWriter, req *http.Request, ps
}, http.StatusForbidden) }, http.StatusForbidden)
return return
} }
q := &orm.Query{ Size: 1} q := &orm.Query{Size: 1}
q.AddSort("timestamp", orm.DESC) q.AddSort("timestamp", orm.DESC)
q.Conds = orm.And( q.Conds = orm.And(
orm.Eq("metadata.category", "elasticsearch"), orm.Eq("metadata.category", "elasticsearch"),
@ -859,13 +857,13 @@ func (h *APIHandler) getIndexNodes(w http.ResponseWriter, req *http.Request, ps
err, result := orm.Search(event.Event{}, q) err, result := orm.Search(event.Event{}, q)
if err != nil { if err != nil {
resBody["error"] = err.Error() resBody["error"] = err.Error()
h.WriteJSON(w,resBody, http.StatusInternalServerError ) h.WriteJSON(w, resBody, http.StatusInternalServerError)
} }
namesM := util.MapStr{} namesM := util.MapStr{}
if len(result.Result) > 0 { if len(result.Result) > 0 {
if data, ok := result.Result[0].(map[string]interface{}); ok { if data, ok := result.Result[0].(map[string]interface{}); ok {
if routingTable, exists := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "index_routing_table"}, data); exists { if routingTable, exists := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "index_routing_table"}, data); exists {
if table, ok := routingTable.(map[string]interface{}); ok{ if table, ok := routingTable.(map[string]interface{}); ok {
if shardsM, ok := table["shards"].(map[string]interface{}); ok { if shardsM, ok := table["shards"].(map[string]interface{}); ok {
for _, rows := range shardsM { for _, rows := range shardsM {
if rowsArr, ok := rows.([]interface{}); ok { if rowsArr, ok := rows.([]interface{}); ok {
@ -887,12 +885,12 @@ func (h *APIHandler) getIndexNodes(w http.ResponseWriter, req *http.Request, ps
} }
//node uuid //node uuid
nodeIds := make([]interface{}, 0, len(namesM) ) nodeIds := make([]interface{}, 0, len(namesM))
for name, _ := range namesM { for name, _ := range namesM {
nodeIds = append(nodeIds, name) nodeIds = append(nodeIds, name)
} }
q1 := &orm.Query{ Size: 100} q1 := &orm.Query{Size: 100}
q1.AddSort("timestamp", orm.DESC) q1.AddSort("timestamp", orm.DESC)
q1.Conds = orm.And( q1.Conds = orm.And(
orm.Eq("metadata.category", "elasticsearch"), orm.Eq("metadata.category", "elasticsearch"),
@ -902,7 +900,7 @@ func (h *APIHandler) getIndexNodes(w http.ResponseWriter, req *http.Request, ps
err, result = orm.Search(elastic.NodeConfig{}, q1) err, result = orm.Search(elastic.NodeConfig{}, q1)
if err != nil { if err != nil {
resBody["error"] = err.Error() resBody["error"] = err.Error()
h.WriteJSON(w,resBody, http.StatusInternalServerError ) h.WriteJSON(w, resBody, http.StatusInternalServerError)
} }
nodes := []interface{}{} nodes := []interface{}{}
for _, hit := range result.Result { for _, hit := range result.Result {
@ -947,7 +945,7 @@ func (h APIHandler) ListIndex(w http.ResponseWriter, req *http.Request, ps httpr
} }
var must = []util.MapStr{} var must = []util.MapStr{}
if !util.StringInArray(ids, "*"){ if !util.StringInArray(ids, "*") {
must = append(must, util.MapStr{ must = append(must, util.MapStr{
"terms": util.MapStr{ "terms": util.MapStr{
@ -958,9 +956,8 @@ func (h APIHandler) ListIndex(w http.ResponseWriter, req *http.Request, ps httpr
if keyword != "" { if keyword != "" {
must = append(must, util.MapStr{ must = append(must, util.MapStr{
"wildcard":util.MapStr{ "wildcard": util.MapStr{
"metadata.index_name": "metadata.index_name": util.MapStr{"value": fmt.Sprintf("*%s*", keyword)},
util.MapStr{"value": fmt.Sprintf("*%s*", keyword)},
}, },
}) })
} }
@ -986,7 +983,6 @@ func (h APIHandler) ListIndex(w http.ResponseWriter, req *http.Request, ps httpr
}, },
} }
esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
indexName := orm.GetIndexName(elastic.IndexConfig{}) indexName := orm.GetIndexName(elastic.IndexConfig{})
resp, err := esClient.SearchWithRawQueryDSL(indexName, util.MustToJSONBytes(dsl)) resp, err := esClient.SearchWithRawQueryDSL(indexName, util.MustToJSONBytes(dsl))

11
modules/elastic/api/v1/manage.go
View File

@ -545,7 +545,7 @@ func (h *APIHandler) HandleClusterMetricsAction(w http.ResponseWriter, req *http
minBucketSize, err := GetMetricMinBucketSize(id, metricType) minBucketSize, err := GetMetricMinBucketSize(id, metricType)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
}else{ } else {
metrics[key].MinBucketSize = int64(minBucketSize) metrics[key].MinBucketSize = int64(minBucketSize)
} }
} }
@ -648,7 +648,7 @@ func (h *APIHandler) HandleIndexMetricsAction(w http.ResponseWriter, req *http.R
} }
} }
} }
}else{ } else {
metrics, err = h.getIndexMetrics(ctx, req, id, bucketSize, min, max, indexName, top, key) metrics, err = h.getIndexMetrics(ctx, req, id, bucketSize, min, max, indexName, top, key)
if err != nil { if err != nil {
h.WriteError(w, err, http.StatusInternalServerError) h.WriteError(w, err, http.StatusInternalServerError)
@ -660,7 +660,7 @@ func (h *APIHandler) HandleIndexMetricsAction(w http.ResponseWriter, req *http.R
minBucketSize, err := GetMetricMinBucketSize(id, MetricTypeNodeStats) minBucketSize, err := GetMetricMinBucketSize(id, MetricTypeNodeStats)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
}else{ } else {
metrics[key].MinBucketSize = int64(minBucketSize) metrics[key].MinBucketSize = int64(minBucketSize)
} }
} }
@ -796,10 +796,11 @@ const (
ShardCountMetricKey = "shard_count" ShardCountMetricKey = "shard_count"
CircuitBreakerMetricKey = "circuit_breaker" CircuitBreakerMetricKey = "circuit_breaker"
) )
func (h *APIHandler) GetClusterMetrics(ctx context.Context, id string, bucketSize int, min, max int64, metricKey string) (map[string]*common.MetricItem, error) { func (h *APIHandler) GetClusterMetrics(ctx context.Context, id string, bucketSize int, min, max int64, metricKey string) (map[string]*common.MetricItem, error) {
var ( var (
clusterMetricsResult = map[string]*common.MetricItem {} clusterMetricsResult = map[string]*common.MetricItem{}
err error err error
) )
switch metricKey { switch metricKey {
@ -915,12 +916,14 @@ func (h *APIHandler) getClusterMetricsByKey(ctx context.Context, id string, buck
} }
return h.getSingleMetrics(ctx, clusterMetricItems, query, bucketSize) return h.getSingleMetrics(ctx, clusterMetricItems, query, bucketSize)
} }
const ( const (
IndexThroughputMetricKey = "index_throughput" IndexThroughputMetricKey = "index_throughput"
SearchThroughputMetricKey = "search_throughput" SearchThroughputMetricKey = "search_throughput"
IndexLatencyMetricKey = "index_latency" IndexLatencyMetricKey = "index_latency"
SearchLatencyMetricKey = "search_latency" SearchLatencyMetricKey = "search_latency"
) )
func (h *APIHandler) GetClusterIndexMetrics(ctx context.Context, id string, bucketSize int, min, max int64, metricKey string) (map[string]*common.MetricItem, error) { func (h *APIHandler) GetClusterIndexMetrics(ctx context.Context, id string, bucketSize int, min, max int64, metricKey string) (map[string]*common.MetricItem, error) {
bucketSizeStr := fmt.Sprintf("%vs", bucketSize) bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
metricItems := []*common.MetricItem{} metricItems := []*common.MetricItem{}

5
modules/elastic/api/v1/metrics_util.go
View File

@ -233,7 +233,8 @@ const (
MetricTypeNodeStats = "node_stats" MetricTypeNodeStats = "node_stats"
MetricTypeIndexStats = "index_stats" MetricTypeIndexStats = "index_stats"
) )
//GetMetricMinBucketSize returns twice the metrics collection interval based on the cluster ID and metric type
// GetMetricMinBucketSize returns twice the metrics collection interval based on the cluster ID and metric type
func GetMetricMinBucketSize(clusterID, metricType string) (int, error) { func GetMetricMinBucketSize(clusterID, metricType string) (int, error) {
meta := elastic.GetMetadata(clusterID) meta := elastic.GetMetadata(clusterID)
if meta == nil { if meta == nil {
@ -301,7 +302,7 @@ func (h *APIHandler) GetMetricRangeAndBucketSize(req *http.Request, clusterID, m
if err != nil { if err != nil {
return 0, 0, 0, fmt.Errorf("failed to get min bucket size for cluster [%s]:%w", clusterID, err) return 0, 0, 0, fmt.Errorf("failed to get min bucket size for cluster [%s]:%w", clusterID, err)
} }
}else{ } else {
//default to 20 //default to 20
minBucketSize = 20 minBucketSize = 20
} }

205
modules/elastic/api/v1/node_overview.go
View File

@ -45,8 +45,8 @@ import (
) )
func (h *APIHandler) SearchNodeMetadata(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *APIHandler) SearchNodeMetadata(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody:=util.MapStr{} resBody := util.MapStr{}
reqBody := struct{ reqBody := struct {
Keyword string `json:"keyword"` Keyword string `json:"keyword"`
Size int `json:"size"` Size int `json:"size"`
From int `json:"from"` From int `json:"from"`
@ -59,7 +59,7 @@ func (h *APIHandler) SearchNodeMetadata(w http.ResponseWriter, req *http.Request
err := h.DecodeJSON(req, &reqBody) err := h.DecodeJSON(req, &reqBody)
if err != nil { if err != nil {
resBody["error"] = err.Error() resBody["error"] = err.Error()
h.WriteJSON(w,resBody, http.StatusInternalServerError ) h.WriteJSON(w, resBody, http.StatusInternalServerError)
return return
} }
aggs := elastic.BuildSearchTermAggregations(reqBody.Aggregations) aggs := elastic.BuildSearchTermAggregations(reqBody.Aggregations)
@ -77,8 +77,8 @@ func (h *APIHandler) SearchNodeMetadata(w http.ResponseWriter, req *http.Request
}, },
}, },
} }
var should =[]util.MapStr{} var should = []util.MapStr{}
if reqBody.SearchField != ""{ if reqBody.SearchField != "" {
should = []util.MapStr{ should = []util.MapStr{
{ {
"prefix": util.MapStr{ "prefix": util.MapStr{
@ -101,7 +101,7 @@ func (h *APIHandler) SearchNodeMetadata(w http.ResponseWriter, req *http.Request
}, },
}, },
} }
}else{ } else {
should = []util.MapStr{ should = []util.MapStr{
{ {
"prefix": util.MapStr{ "prefix": util.MapStr{
@ -143,19 +143,14 @@ func (h *APIHandler) SearchNodeMetadata(w http.ResponseWriter, req *http.Request
} }
clusterFilter, hasPrivilege := h.GetClusterFilter(req, "metadata.cluster_id") clusterFilter, hasPrivilege := h.GetClusterFilter(req, "metadata.cluster_id")
if !hasPrivilege && clusterFilter == nil { if !hasPrivilege && clusterFilter == nil {
h.WriteJSON(w, elastic.SearchResponse{ h.WriteJSON(w, elastic.SearchResponse{}, http.StatusOK)
}, http.StatusOK)
return return
} }
must := []interface{}{ must := []interface{}{}
}
if !hasPrivilege && clusterFilter != nil { if !hasPrivilege && clusterFilter != nil {
must = append(must, clusterFilter) must = append(must, clusterFilter)
} }
query := util.MapStr{ query := util.MapStr{
"aggs": aggs, "aggs": aggs,
"size": reqBody.Size, "size": reqBody.Size,
@ -190,7 +185,7 @@ func (h *APIHandler) SearchNodeMetadata(w http.ResponseWriter, req *http.Request
response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(orm.GetIndexName(elastic.NodeConfig{}), dsl) response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(orm.GetIndexName(elastic.NodeConfig{}), dsl)
if err != nil { if err != nil {
resBody["error"] = err.Error() resBody["error"] = err.Error()
h.WriteJSON(w,resBody, http.StatusInternalServerError ) h.WriteJSON(w, resBody, http.StatusInternalServerError)
return return
} }
w.Write(util.MustToJSONBytes(response)) w.Write(util.MustToJSONBytes(response))
@ -316,10 +311,10 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps
return return
} }
// 索引速率 // 索引速率
indexMetric:=newMetricItem("indexing", 1, OperationGroupKey) indexMetric := newMetricItem("indexing", 1, OperationGroupKey)
indexMetric.AddAxi("indexing rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) indexMetric.AddAxi("indexing rate", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems := []GroupMetricItem{} nodeMetricItems := []GroupMetricItem{}
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "indexing", Key: "indexing",
Field: "payload.elasticsearch.node_stats.indices.indexing.index_total", Field: "payload.elasticsearch.node_stats.indices.indexing.index_total",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -328,9 +323,9 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps
FormatType: "num", FormatType: "num",
Units: "Indexing/s", Units: "Indexing/s",
}) })
queryMetric:=newMetricItem("search", 2, OperationGroupKey) queryMetric := newMetricItem("search", 2, OperationGroupKey)
queryMetric.AddAxi("query rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) queryMetric.AddAxi("query rate", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ nodeMetricItems = append(nodeMetricItems, GroupMetricItem{
Key: "search", Key: "search",
Field: "payload.elasticsearch.node_stats.indices.search.query_total", Field: "payload.elasticsearch.node_stats.indices.search.query_total",
ID: util.GetUUID(), ID: util.GetUUID(),
@ -340,9 +335,9 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps
Units: "Search/s", Units: "Search/s",
}) })
aggs:=map[string]interface{}{} aggs := map[string]interface{}{}
query=map[string]interface{}{} query = map[string]interface{}{}
query["query"]=util.MapStr{ query["query"] = util.MapStr{
"bool": util.MapStr{ "bool": util.MapStr{
"must": []util.MapStr{ "must": []util.MapStr{
{ {
@ -378,15 +373,15 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps
}, },
} }
for _,metricItem:=range nodeMetricItems{ for _, metricItem := range nodeMetricItems {
aggs[metricItem.ID]=util.MapStr{ aggs[metricItem.ID] = util.MapStr{
"max":util.MapStr{ "max": util.MapStr{
"field": metricItem.Field, "field": metricItem.Field,
}, },
} }
if metricItem.IsDerivative{ if metricItem.IsDerivative {
aggs[metricItem.ID+"_deriv"]=util.MapStr{ aggs[metricItem.ID+"_deriv"] = util.MapStr{
"derivative":util.MapStr{ "derivative": util.MapStr{
"buckets_path": metricItem.ID, "buckets_path": metricItem.ID,
}, },
} }
@ -398,8 +393,8 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps
if err != nil { if err != nil {
panic(err) panic(err)
} }
query["size"]=0 query["size"] = 0
query["aggs"]= util.MapStr{ query["aggs"] = util.MapStr{
"group_by_level": util.MapStr{ "group_by_level": util.MapStr{
"terms": util.MapStr{ "terms": util.MapStr{
"field": "metadata.labels.node_id", "field": "metadata.labels.node_id",
@ -407,11 +402,11 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps
}, },
"aggs": util.MapStr{ "aggs": util.MapStr{
"dates": util.MapStr{ "dates": util.MapStr{
"date_histogram":util.MapStr{ "date_histogram": util.MapStr{
"field": "timestamp", "field": "timestamp",
intervalField: bucketSizeStr, intervalField: bucketSizeStr,
}, },
"aggs":aggs, "aggs": aggs,
}, },
}, },
}, },
@ -425,9 +420,8 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps
indexMetrics := map[string]util.MapStr{} indexMetrics := map[string]util.MapStr{}
for key, item := range metrics { for key, item := range metrics {
for _, line := range item.Lines { for _, line := range item.Lines {
if _, ok := indexMetrics[line.Metric.Label]; !ok{ if _, ok := indexMetrics[line.Metric.Label]; !ok {
indexMetrics[line.Metric.Label] = util.MapStr{ indexMetrics[line.Metric.Label] = util.MapStr{}
}
} }
indexMetrics[line.Metric.Label][key] = line.Data indexMetrics[line.Metric.Label][key] = line.Data
} }
@ -512,7 +506,7 @@ func (h *APIHandler) GetNodeInfo(w http.ResponseWriter, req *http.Request, ps ht
tt, _ := time.Parse(time.RFC3339, ts) tt, _ := time.Parse(time.RFC3339, ts)
if time.Now().Sub(tt).Seconds() > 30 { if time.Now().Sub(tt).Seconds() > 30 {
kvs["status"] = "unavailable" kvs["status"] = "unavailable"
}else{ } else {
kvs["status"] = "available" kvs["status"] = "available"
} }
} }
@ -530,7 +524,7 @@ func (h *APIHandler) GetNodeInfo(w http.ResponseWriter, req *http.Request, ps ht
jvm, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "jvm"}, vresult) jvm, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "jvm"}, vresult)
if ok { if ok {
if jvmVal, ok := jvm.(map[string]interface{});ok { if jvmVal, ok := jvm.(map[string]interface{}); ok {
kvs["jvm"] = util.MapStr{ kvs["jvm"] = util.MapStr{
"mem": jvmVal["mem"], "mem": jvmVal["mem"],
"uptime": jvmVal["uptime_in_millis"], "uptime": jvmVal["uptime_in_millis"],
@ -553,7 +547,7 @@ func (h *APIHandler) GetNodeInfo(w http.ResponseWriter, req *http.Request, ps ht
} }
} }
} }
if len( response.Hits.Hits) > 0 { if len(response.Hits.Hits) > 0 {
hit := response.Hits.Hits[0] hit := response.Hits.Hits[0]
innerMetaData, _ := util.GetMapValueByKeys([]string{"metadata", "labels"}, hit.Source) innerMetaData, _ := util.GetMapValueByKeys([]string{"metadata", "labels"}, hit.Source)
if mp, ok := innerMetaData.(map[string]interface{}); ok { if mp, ok := innerMetaData.(map[string]interface{}); ok {
@ -583,8 +577,8 @@ func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Reque
nodeID := ps.MustGetParameter("node_id") nodeID := ps.MustGetParameter("node_id")
var must = []util.MapStr{ var must = []util.MapStr{
{ {
"term":util.MapStr{ "term": util.MapStr{
"metadata.labels.cluster_uuid":util.MapStr{ "metadata.labels.cluster_uuid": util.MapStr{
"value": clusterUUID, "value": clusterUUID,
}, },
}, },
@ -612,15 +606,15 @@ func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Reque
}, },
} }
resBody := map[string]interface{}{} resBody := map[string]interface{}{}
bucketSize, min, max, err := h.GetMetricRangeAndBucketSize(req,clusterID, MetricTypeNodeStats,60) bucketSize, min, max, err := h.GetMetricRangeAndBucketSize(req, clusterID, MetricTypeNodeStats, 60)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
resBody["error"] = err resBody["error"] = err
h.WriteJSON(w, resBody, http.StatusInternalServerError) h.WriteJSON(w, resBody, http.StatusInternalServerError)
return return
} }
query:=map[string]interface{}{} query := map[string]interface{}{}
query["query"]=util.MapStr{ query["query"] = util.MapStr{
"bool": util.MapStr{ "bool": util.MapStr{
"must": must, "must": must,
"filter": []util.MapStr{ "filter": []util.MapStr{
@ -636,67 +630,67 @@ func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Reque
}, },
} }
bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
metricItems:=[]*common.MetricItem{} metricItems := []*common.MetricItem{}
metricItem:=newMetricItem("cpu", 1, SystemGroupKey) metricItem := newMetricItem("cpu", 1, SystemGroupKey)
metricItem.AddAxi("cpu","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true) metricItem.AddAxi("cpu", "group1", common.PositionLeft, "ratio", "0.[0]", "0.[0]", 5, true)
metricItem.AddLine("Process CPU","Process CPU","process cpu used percent of node.","group1","payload.elasticsearch.node_stats.process.cpu.percent","max",bucketSizeStr,"%","num","0,0.[00]","0,0.[00]",false,false) metricItem.AddLine("Process CPU", "Process CPU", "process cpu used percent of node.", "group1", "payload.elasticsearch.node_stats.process.cpu.percent", "max", bucketSizeStr, "%", "num", "0,0.[00]", "0,0.[00]", false, false)
metricItem.AddLine("OS CPU","OS CPU","process cpu used percent of node.","group1","payload.elasticsearch.node_stats.os.cpu.percent","max",bucketSizeStr,"%","num","0,0.[00]","0,0.[00]",false,false) metricItem.AddLine("OS CPU", "OS CPU", "process cpu used percent of node.", "group1", "payload.elasticsearch.node_stats.os.cpu.percent", "max", bucketSizeStr, "%", "num", "0,0.[00]", "0,0.[00]", false, false)
metricItems=append(metricItems,metricItem) metricItems = append(metricItems, metricItem)
metricItem =newMetricItem("jvm", 2, SystemGroupKey) metricItem = newMetricItem("jvm", 2, SystemGroupKey)
metricItem.AddAxi("JVM Heap","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) metricItem.AddAxi("JVM Heap", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true)
metricItem.AddLine("Max Heap","Max Heap","JVM max Heap of node.","group1","payload.elasticsearch.node_stats.jvm.mem.heap_max_in_bytes","max",bucketSizeStr,"","bytes","0,0.[00]","0,0.[00]",false,false) metricItem.AddLine("Max Heap", "Max Heap", "JVM max Heap of node.", "group1", "payload.elasticsearch.node_stats.jvm.mem.heap_max_in_bytes", "max", bucketSizeStr, "", "bytes", "0,0.[00]", "0,0.[00]", false, false)
metricItem.AddLine("Used Heap","Used Heap","JVM used Heap of node.","group1","payload.elasticsearch.node_stats.jvm.mem.heap_used_in_bytes","max",bucketSizeStr,"","bytes","0,0.[00]","0,0.[00]",false,false) metricItem.AddLine("Used Heap", "Used Heap", "JVM used Heap of node.", "group1", "payload.elasticsearch.node_stats.jvm.mem.heap_used_in_bytes", "max", bucketSizeStr, "", "bytes", "0,0.[00]", "0,0.[00]", false, false)
metricItems=append(metricItems,metricItem) metricItems = append(metricItems, metricItem)
metricItem=newMetricItem("index_throughput", 3, OperationGroupKey) metricItem = newMetricItem("index_throughput", 3, OperationGroupKey)
metricItem.AddAxi("indexing","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) metricItem.AddAxi("indexing", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
metricItem.AddLine("Indexing Rate","Total Shards","Number of documents being indexed for node.","group1","payload.elasticsearch.node_stats.indices.indexing.index_total","max",bucketSizeStr,"doc/s","num","0,0.[00]","0,0.[00]",false,true) metricItem.AddLine("Indexing Rate", "Total Shards", "Number of documents being indexed for node.", "group1", "payload.elasticsearch.node_stats.indices.indexing.index_total", "max", bucketSizeStr, "doc/s", "num", "0,0.[00]", "0,0.[00]", false, true)
metricItems=append(metricItems,metricItem) metricItems = append(metricItems, metricItem)
metricItem=newMetricItem("search_throughput", 4, OperationGroupKey) metricItem = newMetricItem("search_throughput", 4, OperationGroupKey)
metricItem.AddAxi("searching","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,false) metricItem.AddAxi("searching", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
metricItem.AddLine("Search Rate","Total Shards", metricItem.AddLine("Search Rate", "Total Shards",
"Number of search requests being executed.", "Number of search requests being executed.",
"group1","payload.elasticsearch.node_stats.indices.search.query_total","max",bucketSizeStr,"query/s","num","0,0.[00]","0,0.[00]",false,true) "group1", "payload.elasticsearch.node_stats.indices.search.query_total", "max", bucketSizeStr, "query/s", "num", "0,0.[00]", "0,0.[00]", false, true)
metricItems=append(metricItems,metricItem) metricItems = append(metricItems, metricItem)
metricItem=newMetricItem("index_latency", 5, LatencyGroupKey) metricItem = newMetricItem("index_latency", 5, LatencyGroupKey)
metricItem.AddAxi("indexing","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) metricItem.AddAxi("indexing", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true)
metricItem.AddLine("Indexing","Indexing Latency","Average latency for indexing documents.","group1","payload.elasticsearch.node_stats.indices.indexing.index_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) metricItem.AddLine("Indexing", "Indexing Latency", "Average latency for indexing documents.", "group1", "payload.elasticsearch.node_stats.indices.indexing.index_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.node_stats.indices.indexing.index_total" metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.node_stats.indices.indexing.index_total"
metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 { metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
return value/value2 return value / value2
} }
metricItem.AddLine("Indexing","Delete Latency","Average latency for delete documents.","group1","payload.elasticsearch.node_stats.indices.indexing.delete_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) metricItem.AddLine("Indexing", "Delete Latency", "Average latency for delete documents.", "group1", "payload.elasticsearch.node_stats.indices.indexing.delete_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.node_stats.indices.indexing.delete_total" metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.node_stats.indices.indexing.delete_total"
metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 { metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
return value/value2 return value / value2
} }
metricItems=append(metricItems,metricItem) metricItems = append(metricItems, metricItem)
metricItem=newMetricItem("search_latency", 6, LatencyGroupKey) metricItem = newMetricItem("search_latency", 6, LatencyGroupKey)
metricItem.AddAxi("searching","group2",common.PositionLeft,"num","0,0","0,0.[00]",5,false) metricItem.AddAxi("searching", "group2", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false)
metricItem.AddLine("Searching","Query Latency","Average latency for searching query.","group2","payload.elasticsearch.node_stats.indices.search.query_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) metricItem.AddLine("Searching", "Query Latency", "Average latency for searching query.", "group2", "payload.elasticsearch.node_stats.indices.search.query_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.query_total" metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.query_total"
metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 { metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 {
return value/value2 return value / value2
} }
metricItem.AddLine("Searching","Fetch Latency","Average latency for searching fetch.","group2","payload.elasticsearch.node_stats.indices.search.fetch_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) metricItem.AddLine("Searching", "Fetch Latency", "Average latency for searching fetch.", "group2", "payload.elasticsearch.node_stats.indices.search.fetch_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.fetch_total" metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.fetch_total"
metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 { metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 {
return value/value2 return value / value2
} }
metricItem.AddLine("Searching","Scroll Latency","Average latency for searching fetch.","group2","payload.elasticsearch.node_stats.indices.search.scroll_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) metricItem.AddLine("Searching", "Scroll Latency", "Average latency for searching fetch.", "group2", "payload.elasticsearch.node_stats.indices.search.scroll_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true)
metricItem.Lines[2].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.scroll_total" metricItem.Lines[2].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.scroll_total"
metricItem.Lines[2].Metric.Calc = func(value, value2 float64) float64 { metricItem.Lines[2].Metric.Calc = func(value, value2 float64) float64 {
return value/value2 return value / value2
} }
metricItems=append(metricItems,metricItem) metricItems = append(metricItems, metricItem)
metricItem =newMetricItem("parent_breaker", 8, SystemGroupKey) metricItem = newMetricItem("parent_breaker", 8, SystemGroupKey)
metricItem.AddLine("Parent Breaker Tripped","Parent Breaker Tripped","Rate of the circuit breaker has been triggered and prevented an out of memory error.","group1","payload.elasticsearch.node_stats.breakers.parent.tripped","max",bucketSizeStr,"times/s","num","0,0.[00]","0,0.[00]",false,true) metricItem.AddLine("Parent Breaker Tripped", "Parent Breaker Tripped", "Rate of the circuit breaker has been triggered and prevented an out of memory error.", "group1", "payload.elasticsearch.node_stats.breakers.parent.tripped", "max", bucketSizeStr, "times/s", "num", "0,0.[00]", "0,0.[00]", false, true)
metricItems=append(metricItems,metricItem) metricItems = append(metricItems, metricItem)
metrics, err := h.getSingleMetrics(context.Background(), metricItems,query, bucketSize) metrics, err := h.getSingleMetrics(context.Background(), metricItems, query, bucketSize)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
h.WriteError(w, err, http.StatusInternalServerError) h.WriteError(w, err, http.StatusInternalServerError)
@ -713,8 +707,8 @@ func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Reque
h.WriteJSON(w, resBody, http.StatusOK) h.WriteJSON(w, resBody, http.StatusOK)
} }
func getNodeHealthMetric(query util.MapStr, bucketSize int)(*common.MetricItem, error){ func getNodeHealthMetric(query util.MapStr, bucketSize int) (*common.MetricItem, error) {
bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) bucketSizeStr := fmt.Sprintf("%vs", bucketSize)
intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr)
if err != nil { if err != nil {
return nil, err return nil, err
@ -740,8 +734,8 @@ func getNodeHealthMetric(query util.MapStr, bucketSize int)(*common.MetricItem,
return nil, err return nil, err
} }
metricItem:=newMetricItem("node_health", 0, "") metricItem := newMetricItem("node_health", 0, "")
metricItem.AddLine("Node health","Node Health","","group1","payload.elasticsearch.node_stats.jvm.uptime_in_millis","min",bucketSizeStr,"%","ratio","0.[00]","0.[00]",false,false) metricItem.AddLine("Node health", "Node Health", "", "group1", "payload.elasticsearch.node_stats.jvm.uptime_in_millis", "min", bucketSizeStr, "%", "ratio", "0.[00]", "0.[00]", false, false)
metricData := []interface{}{} metricData := []interface{}{}
if response.StatusCode == 200 { if response.StatusCode == 200 {
@ -770,7 +764,7 @@ func getNodeHealthMetric(query util.MapStr, bucketSize int)(*common.MetricItem,
return metricItem, nil return metricItem, nil
} }
func getNodeOnlineStatusOfRecentDay(nodeIDs []string)(map[string][]interface{}, error){ func getNodeOnlineStatusOfRecentDay(nodeIDs []string) (map[string][]interface{}, error) {
q := orm.Query{ q := orm.Query{
WildcardIndex: true, WildcardIndex: true,
} }
@ -824,7 +818,7 @@ func getNodeOnlineStatusOfRecentDay(nodeIDs []string)(map[string][]interface{},
{ {
"from": "now-4d/d", "from": "now-4d/d",
"to": "now-3d/d", "to": "now-3d/d",
},{ }, {
"from": "now-3d/d", "from": "now-3d/d",
"to": "now-2d/d", "to": "now-2d/d",
}, { }, {
@ -865,7 +859,7 @@ func getNodeOnlineStatusOfRecentDay(nodeIDs []string)(map[string][]interface{},
{ {
"range": util.MapStr{ "range": util.MapStr{
"timestamp": util.MapStr{ "timestamp": util.MapStr{
"gte":"now-15d", "gte": "now-15d",
"lte": "now", "lte": "now",
}, },
}, },
@ -909,7 +903,7 @@ func getNodeOnlineStatusOfRecentDay(nodeIDs []string)(map[string][]interface{},
//mark node status as offline when uptime less than 10m //mark node status as offline when uptime less than 10m
if v, ok := minUptime.(float64); ok && v >= 600000 { if v, ok := minUptime.(float64); ok && v >= 600000 {
recentStatus[nodeKey] = append(recentStatus[nodeKey], []interface{}{bkVal["key"], "online"}) recentStatus[nodeKey] = append(recentStatus[nodeKey], []interface{}{bkVal["key"], "online"})
}else{ } else {
recentStatus[nodeKey] = append(recentStatus[nodeKey], []interface{}{bkVal["key"], "offline"}) recentStatus[nodeKey] = append(recentStatus[nodeKey], []interface{}{bkVal["key"], "offline"})
} }
} }
@ -927,10 +921,10 @@ func (h *APIHandler) getNodeIndices(w http.ResponseWriter, req *http.Request, ps
max = h.GetParameterOrDefault(req, "max", "now") max = h.GetParameterOrDefault(req, "max", "now")
) )
resBody := map[string] interface{}{} resBody := map[string]interface{}{}
id := ps.ByName("id") id := ps.ByName("id")
nodeUUID := ps.ByName("node_id") nodeUUID := ps.ByName("node_id")
q := &orm.Query{ Size: 1} q := &orm.Query{Size: 1}
q.AddSort("timestamp", orm.DESC) q.AddSort("timestamp", orm.DESC)
q.Conds = orm.And( q.Conds = orm.And(
orm.Eq("metadata.category", "elasticsearch"), orm.Eq("metadata.category", "elasticsearch"),
@ -942,16 +936,16 @@ func (h *APIHandler) getNodeIndices(w http.ResponseWriter, req *http.Request, ps
err, result := orm.Search(event.Event{}, q) err, result := orm.Search(event.Event{}, q)
if err != nil { if err != nil {
resBody["error"] = err.Error() resBody["error"] = err.Error()
h.WriteJSON(w,resBody, http.StatusInternalServerError ) h.WriteJSON(w, resBody, http.StatusInternalServerError)
} }
namesM := util.MapStr{} namesM := util.MapStr{}
if len(result.Result) > 0 { if len(result.Result) > 0 {
if data, ok := result.Result[0].(map[string]interface{}); ok { if data, ok := result.Result[0].(map[string]interface{}); ok {
if routingTable, exists := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_routing_table"}, data); exists { if routingTable, exists := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_routing_table"}, data); exists {
if rows, ok := routingTable.([]interface{}); ok{ if rows, ok := routingTable.([]interface{}); ok {
for _, row := range rows { for _, row := range rows {
if v, ok := row.(map[string]interface{}); ok { if v, ok := row.(map[string]interface{}); ok {
if indexName, ok := v["index"].(string); ok{ if indexName, ok := v["index"].(string); ok {
namesM[indexName] = true namesM[indexName] = true
} }
} }
@ -961,12 +955,12 @@ func (h *APIHandler) getNodeIndices(w http.ResponseWriter, req *http.Request, ps
} }
} }
indexNames := make([]interface{}, 0, len(namesM) ) indexNames := make([]interface{}, 0, len(namesM))
for name, _ := range namesM { for name, _ := range namesM {
indexNames = append(indexNames, name) indexNames = append(indexNames, name)
} }
q1 := &orm.Query{ Size: 100} q1 := &orm.Query{Size: 100}
q1.AddSort("timestamp", orm.DESC) q1.AddSort("timestamp", orm.DESC)
q1.Conds = orm.And( q1.Conds = orm.And(
orm.Eq("metadata.category", "elasticsearch"), orm.Eq("metadata.category", "elasticsearch"),
@ -977,13 +971,13 @@ func (h *APIHandler) getNodeIndices(w http.ResponseWriter, req *http.Request, ps
err, result = orm.Search(elastic.IndexConfig{}, q1) err, result = orm.Search(elastic.IndexConfig{}, q1)
if err != nil { if err != nil {
resBody["error"] = err.Error() resBody["error"] = err.Error()
h.WriteJSON(w,resBody, http.StatusInternalServerError ) h.WriteJSON(w, resBody, http.StatusInternalServerError)
} }
indices, err := h.getLatestIndices(req, min, max, id, &result) indices, err := h.getLatestIndices(req, min, max, id, &result)
if err != nil { if err != nil {
resBody["error"] = err.Error() resBody["error"] = err.Error()
h.WriteJSON(w,resBody, http.StatusInternalServerError ) h.WriteJSON(w, resBody, http.StatusInternalServerError)
} }
h.WriteJSON(w, indices, http.StatusOK) h.WriteJSON(w, indices, http.StatusOK)
@ -1069,7 +1063,7 @@ func (h *APIHandler) getLatestIndices(req *http.Request, min string, max string,
} }
indices := []interface{}{} indices := []interface{}{}
var indexPattern *radix.Pattern var indexPattern *radix.Pattern
if !hasAllPrivilege{ if !hasAllPrivilege {
indexPattern = radix.Compile(allowedIndices...) indexPattern = radix.Compile(allowedIndices...)
} }
@ -1102,7 +1096,6 @@ func (h *APIHandler) getLatestIndices(req *http.Request, min string, max string,
return indices, nil return indices, nil
} }
func (h *APIHandler) GetNodeShards(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *APIHandler) GetNodeShards(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
clusterID := ps.MustGetParameter("id") clusterID := ps.MustGetParameter("id")
nodeID := ps.MustGetParameter("node_id") nodeID := ps.MustGetParameter("node_id")
@ -1119,9 +1112,9 @@ func (h *APIHandler) GetNodeShards(w http.ResponseWriter, req *http.Request, ps
q1.AddSort("timestamp", orm.DESC) q1.AddSort("timestamp", orm.DESC)
err, result := orm.Search(&event.Event{}, &q1) err, result := orm.Search(&event.Event{}, &q1)
if err != nil { if err != nil {
h.WriteJSON(w,util.MapStr{ h.WriteJSON(w, util.MapStr{
"error": err.Error(), "error": err.Error(),
}, http.StatusInternalServerError ) }, http.StatusInternalServerError)
return return
} }
var shardInfo interface{} = []interface{}{} var shardInfo interface{} = []interface{}{}

2
modules/security/realm/authc/saml/main.go
View File

@ -28,9 +28,9 @@ import (
"crypto/tls" "crypto/tls"
"crypto/x509" "crypto/x509"
"fmt" "fmt"
"github.com/crewjam/saml"
"net/http" "net/http"
"net/url" "net/url"
"github.com/crewjam/saml"
"github.com/crewjam/saml/samlsp" "github.com/crewjam/saml/samlsp"
) )

2
plugin/api/alerting/alert.go
View File

@ -166,7 +166,7 @@ func (h *AlertAPI) getAlertStats(w http.ResponseWriter, req *http.Request, ps ht
}, },
} }
searchRes, err := esClient.SearchWithRawQueryDSL(orm.GetWildcardIndexName(alerting.Alert{}), util.MustToJSONBytes(queryDsl) ) searchRes, err := esClient.SearchWithRawQueryDSL(orm.GetWildcardIndexName(alerting.Alert{}), util.MustToJSONBytes(queryDsl))
if err != nil { if err != nil {
h.WriteJSON(w, util.MapStr{ h.WriteJSON(w, util.MapStr{
"error": err.Error(), "error": err.Error(),

6
plugin/api/alerting/channel.go
View File

@ -219,7 +219,7 @@ func (h *AlertAPI) searchChannel(w http.ResponseWriter, req *http.Request, ps ht
mustQ := []interface{}{} mustQ := []interface{}{}
if keyword != "" { if keyword != "" {
mustQ = append(mustQ, util.MapStr{ mustQ = append(mustQ, util.MapStr{
"query_string": util.MapStr{"default_field":"*","query": keyword}, "query_string": util.MapStr{"default_field": "*", "query": keyword},
}) })
} }
if typ != "" { if typ != "" {
@ -323,8 +323,8 @@ func (h *AlertAPI) testChannel(w http.ResponseWriter, req *http.Request, ps http
"results": []util.MapStr{ "results": []util.MapStr{
{"threshold": "90", {"threshold": "90",
"priority": "critical", "priority": "critical",
"group_values": []string{firstGrpValue, "group_value2" }, "group_values": []string{firstGrpValue, "group_value2"},
"issue_timestamp": time.Now().UnixMilli()-500, "issue_timestamp": time.Now().UnixMilli() - 500,
"result_value": 90, "result_value": 90,
"relation_values": util.MapStr{"a": 100, "b": 90}, "relation_values": util.MapStr{"a": 100, "b": 90},
}, },

24
plugin/api/alerting/message.go
View File

@ -83,7 +83,7 @@ func (h *AlertAPI) ignoreAlertMessage(w http.ResponseWriter, req *http.Request,
}, },
}) })
source = fmt.Sprintf("ctx._source['status'] = '%s'", alerting.MessageStateAlerting) source = fmt.Sprintf("ctx._source['status'] = '%s'", alerting.MessageStateAlerting)
}else { } else {
must = append(must, util.MapStr{ must = append(must, util.MapStr{
"term": util.MapStr{ "term": util.MapStr{
"status": util.MapStr{ "status": util.MapStr{
@ -114,7 +114,6 @@ func (h *AlertAPI) ignoreAlertMessage(w http.ResponseWriter, req *http.Request,
_ = kv.DeleteKey(alerting2.KVLastMessageState, []byte(msg.RuleID)) _ = kv.DeleteKey(alerting2.KVLastMessageState, []byte(msg.RuleID))
} }
h.WriteJSON(w, util.MapStr{ h.WriteJSON(w, util.MapStr{
"ids": messageIDs, "ids": messageIDs,
"result": "updated", "result": "updated",
@ -138,7 +137,7 @@ func (h *AlertAPI) getAlertMessageStats(w http.ResponseWriter, req *http.Request
return return
} }
if !hasAllPrivilege { if !hasAllPrivilege {
must = append(must,clusterFilter) must = append(must, clusterFilter)
} }
queryDsl := util.MapStr{ queryDsl := util.MapStr{
"size": 0, "size": 0,
@ -157,7 +156,7 @@ func (h *AlertAPI) getAlertMessageStats(w http.ResponseWriter, req *http.Request
}, },
} }
indexName := orm.GetWildcardIndexName(alerting.AlertMessage{}) indexName := orm.GetWildcardIndexName(alerting.AlertMessage{})
searchRes, err := esClient.SearchWithRawQueryDSL(indexName, util.MustToJSONBytes(queryDsl) ) searchRes, err := esClient.SearchWithRawQueryDSL(indexName, util.MustToJSONBytes(queryDsl))
if err != nil { if err != nil {
h.WriteJSON(w, util.MapStr{ h.WriteJSON(w, util.MapStr{
"error": err.Error(), "error": err.Error(),
@ -172,7 +171,7 @@ func (h *AlertAPI) getAlertMessageStats(w http.ResponseWriter, req *http.Request
} }
} }
} }
for _, status := range []string{"info", "low","medium","high", "critical"} { for _, status := range []string{"info", "low", "medium", "high", "critical"} {
if _, ok := statusCounts[status]; !ok { if _, ok := statusCounts[status]; !ok {
statusCounts[status] = 0 statusCounts[status] = 0
} }
@ -217,7 +216,7 @@ func (h *AlertAPI) getAlertMessageStats(w http.ResponseWriter, req *http.Request
}, },
}, },
} }
searchRes, err = esClient.SearchWithRawQueryDSL(indexName, util.MustToJSONBytes(queryDsl) ) searchRes, err = esClient.SearchWithRawQueryDSL(indexName, util.MustToJSONBytes(queryDsl))
if err != nil { if err != nil {
h.WriteJSON(w, util.MapStr{ h.WriteJSON(w, util.MapStr{
"error": err.Error(), "error": err.Error(),
@ -249,7 +248,6 @@ func (h *AlertAPI) getAlertMessageStats(w http.ResponseWriter, req *http.Request
}, http.StatusOK) }, http.StatusOK)
} }
func (h *AlertAPI) searchAlertMessage(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *AlertAPI) searchAlertMessage(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
var ( var (
@ -292,7 +290,7 @@ func (h *AlertAPI) searchAlertMessage(w http.ResponseWriter, req *http.Request,
}, },
} }
mustBuilder.Write(util.MustToJSONBytes(timeFilter)) mustBuilder.Write(util.MustToJSONBytes(timeFilter))
}else{ } else {
mustBuilder.WriteString(`{"match_all":{}}`) mustBuilder.WriteString(`{"match_all":{}}`)
} }
@ -374,7 +372,7 @@ func (h *AlertAPI) searchAlertMessage(w http.ResponseWriter, req *http.Request,
h.WriteJSON(w, esRes, http.StatusOK) h.WriteJSON(w, esRes, http.StatusOK)
} }
func parseTime( t interface{}, layout string) (time.Time, error){ func parseTime(t interface{}, layout string) (time.Time, error) {
switch t.(type) { switch t.(type) {
case string: case string:
return time.Parse(layout, t.(string)) return time.Parse(layout, t.(string))
@ -417,7 +415,7 @@ func (h *AlertAPI) getAlertMessage(w http.ResponseWriter, req *http.Request, ps
var duration time.Duration var duration time.Duration
if message.Status == alerting.MessageStateRecovered { if message.Status == alerting.MessageStateRecovered {
duration = message.Updated.Sub(message.Created) duration = message.Updated.Sub(message.Created)
}else{ } else {
duration = time.Now().Sub(message.Created) duration = time.Now().Sub(message.Created)
} }
detailObj := util.MapStr{ detailObj := util.MapStr{
@ -497,7 +495,7 @@ func (h *AlertAPI) getMessageNotificationInfo(w http.ResponseWriter, req *http.R
h.WriteJSON(w, notificationInfo, http.StatusOK) h.WriteJSON(w, notificationInfo, http.StatusOK)
} }
func getMessageNotificationStats(msg *alerting.AlertMessage )(util.MapStr, error){ func getMessageNotificationStats(msg *alerting.AlertMessage) (util.MapStr, error) {
rangeQ := util.MapStr{ rangeQ := util.MapStr{
"gte": msg.Created.UnixMilli(), "gte": msg.Created.UnixMilli(),
} }
@ -633,8 +631,8 @@ func extractStatsFromRaw(searchRawRes []byte, grpKey string, actionKey string) [
statsItem["channel_name"] = cn statsItem["channel_name"] = cn
statsItem["error"], _ = jsonparser.GetString(v, "error") statsItem["error"], _ = jsonparser.GetString(v, "error")
} }
}, "top", "hits","hits", "[0]", "_source",actionKey) }, "top", "hits", "hits", "[0]", "_source", actionKey)
statsItem["last_time"], _ = jsonparser.GetString(value, "top", "hits","hits", "[0]", "_source","created") statsItem["last_time"], _ = jsonparser.GetString(value, "top", "hits", "hits", "[0]", "_source", "created")
stats = append(stats, statsItem) stats = append(stats, statsItem)
}, "aggregations", grpKey, "buckets") }, "aggregations", grpKey, "buckets")
return stats return stats

29
plugin/api/index_management/common_command.go
View File

@ -38,8 +38,7 @@ import (
) )
func (h *APIHandler) HandleAddCommonCommandAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *APIHandler) HandleAddCommonCommandAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody := map[string]interface{}{ resBody := map[string]interface{}{}
}
reqParams := elastic.CommonCommand{} reqParams := elastic.CommonCommand{}
err := h.DecodeJSON(req, &reqParams) err := h.DecodeJSON(req, &reqParams)
@ -54,7 +53,7 @@ func (h *APIHandler) HandleAddCommonCommandAction(w http.ResponseWriter, req *ht
reqParams.ID = util.GetUUID() reqParams.ID = util.GetUUID()
esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
queryDSL :=[]byte(fmt.Sprintf(`{"size":1, "query":{"bool":{"must":{"match":{"title.keyword":"%s"}}}}}`, reqParams.Title)) queryDSL := []byte(fmt.Sprintf(`{"size":1, "query":{"bool":{"must":{"match":{"title.keyword":"%s"}}}}}`, reqParams.Title))
var indexName = orm.GetIndexName(reqParams) var indexName = orm.GetIndexName(reqParams)
searchRes, err := esClient.SearchWithRawQueryDSL(indexName, queryDSL) searchRes, err := esClient.SearchWithRawQueryDSL(indexName, queryDSL)
if err != nil { if err != nil {
@ -69,7 +68,7 @@ func (h *APIHandler) HandleAddCommonCommandAction(w http.ResponseWriter, req *ht
h.WriteJSON(w, resBody, http.StatusOK) h.WriteJSON(w, resBody, http.StatusOK)
return return
} }
_, err = esClient.Index(indexName,"", reqParams.ID, reqParams, "wait_for") _, err = esClient.Index(indexName, "", reqParams.ID, reqParams, "wait_for")
if err != nil { if err != nil {
log.Error(err) log.Error(err)
resBody["error"] = err.Error() resBody["error"] = err.Error()
@ -81,12 +80,11 @@ func (h *APIHandler) HandleAddCommonCommandAction(w http.ResponseWriter, req *ht
resBody["result"] = "created" resBody["result"] = "created"
resBody["_source"] = reqParams resBody["_source"] = reqParams
h.WriteJSON(w, resBody,http.StatusOK) h.WriteJSON(w, resBody, http.StatusOK)
} }
func (h *APIHandler) HandleSaveCommonCommandAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *APIHandler) HandleSaveCommonCommandAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody := map[string]interface{}{ resBody := map[string]interface{}{}
}
reqParams := elastic.CommonCommand{} reqParams := elastic.CommonCommand{}
err := h.DecodeJSON(req, &reqParams) err := h.DecodeJSON(req, &reqParams)
@ -99,7 +97,7 @@ func (h *APIHandler) HandleSaveCommonCommandAction(w http.ResponseWriter, req *h
reqParams.ID = ps.ByName("cid") reqParams.ID = ps.ByName("cid")
esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
queryDSL :=[]byte(fmt.Sprintf(`{"size":1, "query":{"bool":{"must":{"match":{"title.keyword":"%s"}}}}}`, reqParams.Title)) queryDSL := []byte(fmt.Sprintf(`{"size":1, "query":{"bool":{"must":{"match":{"title.keyword":"%s"}}}}}`, reqParams.Title))
var indexName = orm.GetIndexName(reqParams) var indexName = orm.GetIndexName(reqParams)
searchRes, err := esClient.SearchWithRawQueryDSL(indexName, queryDSL) searchRes, err := esClient.SearchWithRawQueryDSL(indexName, queryDSL)
if err != nil { if err != nil {
@ -114,7 +112,7 @@ func (h *APIHandler) HandleSaveCommonCommandAction(w http.ResponseWriter, req *h
h.WriteJSON(w, resBody, http.StatusInternalServerError) h.WriteJSON(w, resBody, http.StatusInternalServerError)
return return
} }
_, err = esClient.Index(indexName,"", reqParams.ID, reqParams, "wait_for") _, err = esClient.Index(indexName, "", reqParams.ID, reqParams, "wait_for")
if err != nil { if err != nil {
log.Error(err) log.Error(err)
resBody["error"] = err.Error() resBody["error"] = err.Error()
@ -126,12 +124,11 @@ func (h *APIHandler) HandleSaveCommonCommandAction(w http.ResponseWriter, req *h
resBody["result"] = "updated" resBody["result"] = "updated"
resBody["_source"] = reqParams resBody["_source"] = reqParams
h.WriteJSON(w, resBody,http.StatusOK) h.WriteJSON(w, resBody, http.StatusOK)
} }
func (h *APIHandler) HandleQueryCommonCommandAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *APIHandler) HandleQueryCommonCommandAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
resBody := map[string]interface{}{ resBody := map[string]interface{}{}
}
var ( var (
keyword = h.GetParameterOrDefault(req, "keyword", "") keyword = h.GetParameterOrDefault(req, "keyword", "")
@ -140,7 +137,7 @@ func (h *APIHandler) HandleQueryCommonCommandAction(w http.ResponseWriter, req *
strFrom = h.GetParameterOrDefault(req, "from", "0") strFrom = h.GetParameterOrDefault(req, "from", "0")
filterBuilder = &strings.Builder{} filterBuilder = &strings.Builder{}
) )
if keyword != ""{ if keyword != "" {
filterBuilder.WriteString(fmt.Sprintf(`{"query_string": { filterBuilder.WriteString(fmt.Sprintf(`{"query_string": {
"default_field": "*", "default_field": "*",
"query": "%s" "query": "%s"
@ -167,7 +164,7 @@ func (h *APIHandler) HandleQueryCommonCommandAction(w http.ResponseWriter, req *
return return
} }
h.WriteJSON(w, searchRes,http.StatusOK) h.WriteJSON(w, searchRes, http.StatusOK)
} }
func (h *APIHandler) HandleDeleteCommonCommandAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *APIHandler) HandleDeleteCommonCommandAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
@ -178,9 +175,9 @@ func (h *APIHandler) HandleDeleteCommonCommandAction(w http.ResponseWriter, req
if err != nil { if err != nil {
log.Error(err) log.Error(err)
resBody["error"] = err.Error() resBody["error"] = err.Error()
if delRes!=nil{ if delRes != nil {
h.WriteJSON(w, resBody, delRes.StatusCode) h.WriteJSON(w, resBody, delRes.StatusCode)
}else{ } else {
h.WriteJSON(w, resBody, http.StatusInternalServerError) h.WriteJSON(w, resBody, http.StatusInternalServerError)
} }
return return

4
plugin/api/insight/dashboard.go
View File

@ -28,13 +28,13 @@
package insight package insight
import ( import (
"net/http"
"strconv"
log "github.com/cihub/seelog" log "github.com/cihub/seelog"
insight2 "infini.sh/console/model/insight" insight2 "infini.sh/console/model/insight"
httprouter "infini.sh/framework/core/api/router" httprouter "infini.sh/framework/core/api/router"
"infini.sh/framework/core/orm" "infini.sh/framework/core/orm"
"infini.sh/framework/core/util" "infini.sh/framework/core/util"
"net/http"
"strconv"
) )
func (h *InsightAPI) createDashboard(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *InsightAPI) createDashboard(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {

6
plugin/api/insight/map_label.go
View File

@ -70,8 +70,8 @@ func (h *InsightAPI) renderMapLabelTemplate(w http.ResponseWriter, req *http.Req
kv := strings.Split(part, "=") kv := strings.Split(part, "=")
if len(kv) == 2 { if len(kv) == 2 {
k := strings.TrimSpace(kv[0]) k := strings.TrimSpace(kv[0])
kvs[k]= strings.TrimSpace(kv[1]) kvs[k] = strings.TrimSpace(kv[1])
}else{ } else {
log.Debugf("got unexpected directory part: %s", part) log.Debugf("got unexpected directory part: %s", part)
} }
} }
@ -120,7 +120,7 @@ func (h *InsightAPI) renderMapLabelTemplate(w http.ResponseWriter, req *http.Req
cacheLabels, err = common2.GetLabelMaps(indexName, keyField, valueField, client, keyFieldValues, len(keyFieldValues)) cacheLabels, err = common2.GetLabelMaps(indexName, keyField, valueField, client, keyFieldValues, len(keyFieldValues))
if err != nil { if err != nil {
log.Error(err) log.Error(err)
}else{ } else {
cacheLabelsMap[cacheKey] = cacheLabels cacheLabelsMap[cacheKey] = cacheLabels
} }
} }

4
plugin/api/insight/metadata.go
View File

@ -348,13 +348,13 @@ func getMetricData(metric *insight.Metric) (interface{}, error) {
} }
} }
retMetricDataItem.Timestamp = timestamp retMetricDataItem.Timestamp = timestamp
if len(metric.Formulas) <= 1 && metric.Formula != ""{ if len(metric.Formulas) <= 1 && metric.Formula != "" {
//support older versions by returning the result for a single formula. //support older versions by returning the result for a single formula.
retMetricDataItem.Value = result retMetricDataItem.Value = result
} else { } else {
if v, ok := retMetricDataItem.Value.(map[string]interface{}); ok { if v, ok := retMetricDataItem.Value.(map[string]interface{}); ok {
v[formula] = result v[formula] = result
}else{ } else {
retMetricDataItem.Value = map[string]interface{}{formula: result} retMetricDataItem.Value = map[string]interface{}{formula: result}
} }
} }

10
plugin/api/insight/metric_util.go
View File

@ -131,7 +131,7 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) {
"field": metric.TimeField, "field": metric.TimeField,
"buckets": buckets, "buckets": buckets,
} }
}else{ } else {
dateHistogramAggName = "date_histogram" dateHistogramAggName = "date_histogram"
verInfo := elastic.GetClient(metric.ClusterId).GetVersion() verInfo := elastic.GetClient(metric.ClusterId).GetVersion()
@ -179,7 +179,7 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) {
"field": groups[i].Field, "field": groups[i].Field,
"size": limit, "size": limit,
} }
if i == grpLength - 1 && len(metric.Sort) > 0 { if i == grpLength-1 && len(metric.Sort) > 0 {
//use bucket sort instead of terms order when time after group //use bucket sort instead of terms order when time after group
if !timeBeforeGroup && len(metric.Sort) > 0 { if !timeBeforeGroup && len(metric.Sort) > 0 {
basicAggs["sort_field"] = util.MapStr{ basicAggs["sort_field"] = util.MapStr{
@ -197,7 +197,7 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) {
}, },
}, },
} }
}else{ } else {
var termsOrder []interface{} var termsOrder []interface{}
percentAggs := []string{"p99", "p95", "p90", "p80", "p50"} percentAggs := []string{"p99", "p95", "p90", "p80", "p50"}
for _, sortItem := range metric.Sort { for _, sortItem := range metric.Sort {
@ -288,7 +288,7 @@ func CollectMetricData(agg interface{}, timeBeforeGroup bool) ([]insight.MetricD
} }
// timeBeforeGroup => false // timeBeforeGroup => false
func collectMetricData(agg interface{}, groupValues []string, metricData *[]insight.MetricData) (interval string){ func collectMetricData(agg interface{}, groupValues []string, metricData *[]insight.MetricData) (interval string) {
if aggM, ok := agg.(map[string]interface{}); ok { if aggM, ok := agg.(map[string]interface{}); ok {
if timeBks, ok := aggM["time_buckets"].(map[string]interface{}); ok { if timeBks, ok := aggM["time_buckets"].(map[string]interface{}); ok {
interval, _ = timeBks["interval"].(string) interval, _ = timeBks["interval"].(string)
@ -351,7 +351,7 @@ func collectMetricData(agg interface{}, groupValues []string, metricData *[]insi
} }
// timeBeforeGroup => true // timeBeforeGroup => true
func collectMetricDataOther(agg interface{}, groupValues []string, metricData *[]insight.MetricData, timeKey interface{}) (interval string){ func collectMetricDataOther(agg interface{}, groupValues []string, metricData *[]insight.MetricData, timeKey interface{}) (interval string) {
if aggM, ok := agg.(map[string]interface{}); ok { if aggM, ok := agg.(map[string]interface{}); ok {
if timeBks, ok := aggM["time_buckets"].(map[string]interface{}); ok { if timeBks, ok := aggM["time_buckets"].(map[string]interface{}); ok {
interval, _ = timeBks["interval"].(string) interval, _ = timeBks["interval"].(string)

2
plugin/api/insight/visualization.go
View File

@ -34,8 +34,8 @@ import (
"strings" "strings"
log "github.com/cihub/seelog" log "github.com/cihub/seelog"
httprouter "infini.sh/framework/core/api/router"
"infini.sh/console/model/insight" "infini.sh/console/model/insight"
httprouter "infini.sh/framework/core/api/router"
"infini.sh/framework/core/orm" "infini.sh/framework/core/orm"
"infini.sh/framework/core/util" "infini.sh/framework/core/util"
) )

4
plugin/managed/server/config.go
View File

@ -29,13 +29,13 @@ package server
import ( import (
log "github.com/cihub/seelog" log "github.com/cihub/seelog"
"infini.sh/framework/modules/configs/common"
"infini.sh/framework/modules/configs/config"
httprouter "infini.sh/framework/core/api/router" httprouter "infini.sh/framework/core/api/router"
config3 "infini.sh/framework/core/config" config3 "infini.sh/framework/core/config"
"infini.sh/framework/core/global" "infini.sh/framework/core/global"
"infini.sh/framework/core/model" "infini.sh/framework/core/model"
"infini.sh/framework/core/util" "infini.sh/framework/core/util"
"infini.sh/framework/modules/configs/common"
"infini.sh/framework/modules/configs/config"
"net/http" "net/http"
"path" "path"
"sync" "sync"

2
plugin/managed/server/instance.go
View File

@ -37,13 +37,13 @@ import (
log "github.com/cihub/seelog" log "github.com/cihub/seelog"
"infini.sh/console/core/security/enum" "infini.sh/console/core/security/enum"
"infini.sh/framework/modules/configs/common"
"infini.sh/framework/core/api" "infini.sh/framework/core/api"
httprouter "infini.sh/framework/core/api/router" httprouter "infini.sh/framework/core/api/router"
elastic2 "infini.sh/framework/core/elastic" elastic2 "infini.sh/framework/core/elastic"
"infini.sh/framework/core/model" "infini.sh/framework/core/model"
"infini.sh/framework/core/orm" "infini.sh/framework/core/orm"
"infini.sh/framework/core/util" "infini.sh/framework/core/util"
"infini.sh/framework/modules/configs/common"
"infini.sh/framework/modules/elastic" "infini.sh/framework/modules/elastic"
common2 "infini.sh/framework/modules/elastic/common" common2 "infini.sh/framework/modules/elastic/common"
) )

2
plugin/managed/server/manager.go
View File

@ -32,11 +32,11 @@ import (
"fmt" "fmt"
log "github.com/cihub/seelog" log "github.com/cihub/seelog"
"infini.sh/console/core" "infini.sh/console/core"
"infini.sh/framework/modules/configs/common"
"infini.sh/framework/core/api" "infini.sh/framework/core/api"
"infini.sh/framework/core/errors" "infini.sh/framework/core/errors"
"infini.sh/framework/core/global" "infini.sh/framework/core/global"
"infini.sh/framework/core/util" "infini.sh/framework/core/util"
"infini.sh/framework/modules/configs/common"
"net" "net"
"net/http" "net/http"
"net/url" "net/url"

2
service/alerting/action/email.go
View File

@ -42,7 +42,7 @@ type EmailAction struct {
const EmailQueueName = "email_messages" const EmailQueueName = "email_messages"
func (act *EmailAction) Execute()([]byte, error){ func (act *EmailAction) Execute() ([]byte, error) {
queueCfg := queue.GetOrInitConfig(EmailQueueName) queueCfg := queue.GetOrInitConfig(EmailQueueName)
if act.Data.ServerID == "" { if act.Data.ServerID == "" {
return nil, fmt.Errorf("parameter server_id must not be empty") return nil, fmt.Errorf("parameter server_id must not be empty")

3
service/alerting/action/webhook.go
View File

@ -50,7 +50,7 @@ var actionClient = http.Client{
}, },
} }
func (act *WebhookAction) Execute()([]byte, error){ func (act *WebhookAction) Execute() ([]byte, error) {
var reqURL = act.Data.URL var reqURL = act.Data.URL
reqBody := strings.NewReader(act.Message) reqBody := strings.NewReader(act.Message)
req, err := http.NewRequest(http.MethodPost, reqURL, reqBody) req, err := http.NewRequest(http.MethodPost, reqURL, reqBody)
@ -67,4 +67,3 @@ func (act *WebhookAction) Execute()([]byte, error){
defer res.Body.Close() defer res.Body.Close()
return ioutil.ReadAll(res.Body) return ioutil.ReadAll(res.Body)
} }

8
service/alerting/common/helper.go
View File

@ -84,10 +84,10 @@ func PerformChannel(channel *alerting.Channel, ctx map[string]interface{}) ([]by
return executeResult, err, message return executeResult, err, message
} }
func ResolveMessage(messageTemplate string, ctx map[string]interface{}) ([]byte, error){ func ResolveMessage(messageTemplate string, ctx map[string]interface{}) ([]byte, error) {
msg := messageTemplate msg := messageTemplate
tmpl, err := template.New("alert-message").Funcs(funcs.GenericFuncMap()).Parse(msg) tmpl, err := template.New("alert-message").Funcs(funcs.GenericFuncMap()).Parse(msg)
if err !=nil { if err != nil {
return nil, fmt.Errorf("parse message temlate error: %w", err) return nil, fmt.Errorf("parse message temlate error: %w", err)
} }
msgBuffer := &bytes.Buffer{} msgBuffer := &bytes.Buffer{}
@ -120,14 +120,14 @@ func RetrieveChannel(ch *alerting.Channel, raiseChannelEnabledErr bool) (*alerti
case alerting.ChannelEmail: case alerting.ChannelEmail:
if ch.Email == nil { if ch.Email == nil {
ch.Email = refCh.Email ch.Email = refCh.Email
}else{ } else {
ch.Email.ServerID = refCh.Email.ServerID ch.Email.ServerID = refCh.Email.ServerID
ch.Email.Recipients = refCh.Email.Recipients ch.Email.Recipients = refCh.Email.Recipients
} }
case alerting.ChannelWebhook: case alerting.ChannelWebhook:
if ch.Webhook == nil { if ch.Webhook == nil {
ch.Webhook = refCh.Webhook ch.Webhook = refCh.Webhook
}else { } else {
ch.Webhook.URL = refCh.Webhook.URL ch.Webhook.URL = refCh.Webhook.URL
} }
} }

6
service/alerting/constants.go
View File

@ -34,7 +34,6 @@ const (
KVLastMessageState = "alert_last_message_state" KVLastMessageState = "alert_last_message_state"
) )
const ( const (
ParamRuleID = "rule_id" //规则 UUID ParamRuleID = "rule_id" //规则 UUID
ParamResourceID = "resource_id" // 资源 UUID ParamResourceID = "resource_id" // 资源 UUID
@ -50,6 +49,7 @@ const (
ParamGroupValues = "group_values" ParamGroupValues = "group_values"
ParamIssueTimestamp = "issue_timestamp" ParamIssueTimestamp = "issue_timestamp"
ParamRelationValues = "relation_values" ParamRelationValues = "relation_values"
//rule expression, rule_id, resource_id, resource_name, event_id, condition_name, preset_value,[group_tags, check_values],
//check_status ,timestamp, // rule expression, rule_id, resource_id, resource_name, event_id, condition_name, preset_value,[group_tags, check_values],
// check_status ,timestamp,
) )

157
service/alerting/elasticsearch/engine.go
View File

@ -34,10 +34,10 @@ import (
log "github.com/cihub/seelog" log "github.com/cihub/seelog"
"infini.sh/console/model" "infini.sh/console/model"
"infini.sh/console/model/alerting" "infini.sh/console/model/alerting"
"infini.sh/console/model/insight"
alerting2 "infini.sh/console/service/alerting" alerting2 "infini.sh/console/service/alerting"
"infini.sh/console/service/alerting/common" "infini.sh/console/service/alerting/common"
"infini.sh/framework/core/elastic" "infini.sh/framework/core/elastic"
"infini.sh/console/model/insight"
"infini.sh/framework/core/kv" "infini.sh/framework/core/kv"
"infini.sh/framework/core/orm" "infini.sh/framework/core/orm"
"infini.sh/framework/core/util" "infini.sh/framework/core/util"
@ -50,14 +50,14 @@ import (
) )
type Engine struct { type Engine struct {
} }
//GenerateQuery generate a final elasticsearch query dsl object
//when RawFilter of rule is not empty, priority use it, otherwise to covert from Filter of rule (todo) // GenerateQuery generate a final elasticsearch query dsl object
//auto generate time filter query and then attach to final query // when RawFilter of rule is not empty, priority use it, otherwise to covert from Filter of rule (todo)
//auto generate elasticsearch aggregations by metrics of rule // auto generate time filter query and then attach to final query
//group of metric item converted to terms aggregation and TimeField of rule converted to date_histogram aggregation // auto generate elasticsearch aggregations by metrics of rule
//convert statistic of metric item to elasticsearch aggregation // group of metric item converted to terms aggregation and TimeField of rule converted to date_histogram aggregation
// convert statistic of metric item to elasticsearch aggregation
func (engine *Engine) GenerateQuery(rule *alerting.Rule, filterParam *alerting.FilterParam) (interface{}, error) { func (engine *Engine) GenerateQuery(rule *alerting.Rule, filterParam *alerting.FilterParam) (interface{}, error) {
filter, err := engine.GenerateRawFilter(rule, filterParam) filter, err := engine.GenerateRawFilter(rule, filterParam)
if err != nil { if err != nil {
@ -84,11 +84,11 @@ func (engine *Engine) GenerateQuery(rule *alerting.Rule, filterParam *alerting.F
periodInterval = filterParam.BucketSize periodInterval = filterParam.BucketSize
} }
if verInfo.Number==""{ if verInfo.Number == "" {
panic("invalid version") panic("invalid version")
} }
intervalField, err := elastic.GetDateHistogramIntervalField(verInfo.Distribution,verInfo.Number, periodInterval ) intervalField, err := elastic.GetDateHistogramIntervalField(verInfo.Distribution, verInfo.Number, periodInterval)
if err != nil { if err != nil {
return nil, fmt.Errorf("get interval field error: %w", err) return nil, fmt.Errorf("get interval field error: %w", err)
} }
@ -107,7 +107,7 @@ func (engine *Engine) GenerateQuery(rule *alerting.Rule, filterParam *alerting.F
if grpLength := len(groups); grpLength > 0 { if grpLength := len(groups); grpLength > 0 {
var lastGroupAgg util.MapStr var lastGroupAgg util.MapStr
for i := grpLength-1; i>=0; i-- { for i := grpLength - 1; i >= 0; i-- {
limit := groups[i].Limit limit := groups[i].Limit
//top group 10 //top group 10
if limit <= 0 { if limit <= 0 {
@ -124,7 +124,7 @@ func (engine *Engine) GenerateQuery(rule *alerting.Rule, filterParam *alerting.F
groupAgg["aggs"] = util.MapStr{ groupAgg["aggs"] = util.MapStr{
groupID: lastGroupAgg, groupID: lastGroupAgg,
} }
}else{ } else {
groupAgg["aggs"] = timeAggs groupAgg["aggs"] = timeAggs
} }
lastGroupAgg = groupAgg lastGroupAgg = groupAgg
@ -132,7 +132,7 @@ func (engine *Engine) GenerateQuery(rule *alerting.Rule, filterParam *alerting.F
rootAggs = util.MapStr{ rootAggs = util.MapStr{
util.GetUUID(): lastGroupAgg, util.GetUUID(): lastGroupAgg,
} }
}else{ } else {
rootAggs = timeAggs rootAggs = timeAggs
} }
if len(filter) > 0 { if len(filter) > 0 {
@ -150,8 +150,9 @@ func (engine *Engine) GenerateQuery(rule *alerting.Rule, filterParam *alerting.F
"aggs": rootAggs, "aggs": rootAggs,
}, nil }, nil
} }
//generateAgg convert statistic of metric item to elasticsearch aggregation
func (engine *Engine) generateAgg(metricItem *insight.MetricItem) map[string]interface{}{ // generateAgg convert statistic of metric item to elasticsearch aggregation
func (engine *Engine) generateAgg(metricItem *insight.MetricItem) map[string]interface{} {
var ( var (
aggType = "value_count" aggType = "value_count"
field = metricItem.Field field = metricItem.Field
@ -171,7 +172,7 @@ func (engine *Engine) generateAgg(metricItem *insight.MetricItem) map[string]int
isPipeline = true isPipeline = true
case "medium": // from es version 6.6 case "medium": // from es version 6.6
aggType = "median_absolute_deviation" aggType = "median_absolute_deviation"
case "p99", "p95","p90","p80","p50": case "p99", "p95", "p90", "p80", "p50":
aggType = "percentiles" aggType = "percentiles"
percentStr := strings.TrimPrefix(metricItem.Statistic, "p") percentStr := strings.TrimPrefix(metricItem.Statistic, "p")
percent, _ = strconv.ParseFloat(percentStr, 32) percent, _ = strconv.ParseFloat(percentStr, 32)
@ -187,7 +188,7 @@ func (engine *Engine) generateAgg(metricItem *insight.MetricItem) map[string]int
aggType: aggValue, aggType: aggValue,
}, },
} }
if !isPipeline{ if !isPipeline {
return aggs return aggs
} }
pipelineAggID := util.GetUUID() pipelineAggID := util.GetUUID()
@ -200,8 +201,8 @@ func (engine *Engine) generateAgg(metricItem *insight.MetricItem) map[string]int
return aggs return aggs
} }
func (engine *Engine) ConvertFilterQueryToDsl(fq *alerting.FilterQuery) (map[string]interface{}, error){ func (engine *Engine) ConvertFilterQueryToDsl(fq *alerting.FilterQuery) (map[string]interface{}, error) {
if !fq.IsComplex(){ if !fq.IsComplex() {
q := map[string]interface{}{} q := map[string]interface{}{}
if len(fq.Values) == 0 { if len(fq.Values) == 0 {
return nil, fmt.Errorf("values should not be empty") return nil, fmt.Errorf("values should not be empty")
@ -267,14 +268,14 @@ func (engine *Engine) ConvertFilterQueryToDsl(fq *alerting.FilterQuery) (map[str
filterQueries []alerting.FilterQuery filterQueries []alerting.FilterQuery
) )
if len(fq.Not) >0 { if len(fq.Not) > 0 {
boolOperator = "must_not" boolOperator = "must_not"
filterQueries = fq.Not filterQueries = fq.Not
}else if len(fq.Or) > 0 { } else if len(fq.Or) > 0 {
boolOperator = "should" boolOperator = "should"
filterQueries = fq.Or filterQueries = fq.Or
}else { } else {
boolOperator = "must" boolOperator = "must"
filterQueries = fq.And filterQueries = fq.And
} }
@ -299,7 +300,7 @@ func (engine *Engine) ConvertFilterQueryToDsl(fq *alerting.FilterQuery) (map[str
return resultQuery, nil return resultQuery, nil
} }
func getQueryTimeRange(rule *alerting.Rule, filterParam *alerting.FilterParam) (start, end interface{}){ func getQueryTimeRange(rule *alerting.Rule, filterParam *alerting.FilterParam) (start, end interface{}) {
var ( var (
timeStart interface{} timeStart interface{}
timeEnd interface{} timeEnd interface{}
@ -307,7 +308,7 @@ func getQueryTimeRange(rule *alerting.Rule, filterParam *alerting.FilterParam) (
if filterParam != nil { if filterParam != nil {
timeStart = filterParam.Start timeStart = filterParam.Start
timeEnd = filterParam.End timeEnd = filterParam.End
}else{ } else {
var ( var (
units string units string
value int value int
@ -316,23 +317,23 @@ func getQueryTimeRange(rule *alerting.Rule, filterParam *alerting.FilterParam) (
if err != nil { if err != nil {
return nil, fmt.Errorf("parse bucket size of rule [%s] error: %v", rule.Name, err) return nil, fmt.Errorf("parse bucket size of rule [%s] error: %v", rule.Name, err)
} }
if intervalDuration / time.Hour >= 1 { if intervalDuration/time.Hour >= 1 {
units = "h" units = "h"
value = int(intervalDuration / time.Hour) value = int(intervalDuration / time.Hour)
}else if intervalDuration / time.Minute >= 1{ } else if intervalDuration/time.Minute >= 1 {
units = "m" units = "m"
value = int(intervalDuration / time.Minute) value = int(intervalDuration / time.Minute)
}else if intervalDuration / time.Second >= 1 { } else if intervalDuration/time.Second >= 1 {
units = "s" units = "s"
value = int(intervalDuration / time.Second) value = int(intervalDuration / time.Second)
}else{ } else {
return nil, fmt.Errorf("period interval: %s is too small", rule.Metrics.BucketSize) return nil, fmt.Errorf("period interval: %s is too small", rule.Metrics.BucketSize)
} }
bucketCount := rule.Conditions.GetMinimumPeriodMatch() + 1 bucketCount := rule.Conditions.GetMinimumPeriodMatch() + 1
if bucketCount <= 0 { if bucketCount <= 0 {
bucketCount = 1 bucketCount = 1
} }
duration, err := time.ParseDuration(fmt.Sprintf("%d%s", value * bucketCount, units)) duration, err := time.ParseDuration(fmt.Sprintf("%d%s", value*bucketCount, units))
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -342,7 +343,7 @@ func getQueryTimeRange(rule *alerting.Rule, filterParam *alerting.FilterParam) (
return timeStart, timeEnd return timeStart, timeEnd
} }
func (engine *Engine) generateTimeFilter(rule *alerting.Rule, filterParam *alerting.FilterParam) (map[string]interface{}, error){ func (engine *Engine) generateTimeFilter(rule *alerting.Rule, filterParam *alerting.FilterParam) (map[string]interface{}, error) {
timeStart, timeEnd := getQueryTimeRange(rule, filterParam) timeStart, timeEnd := getQueryTimeRange(rule, filterParam)
timeQuery := util.MapStr{ timeQuery := util.MapStr{
"range": util.MapStr{ "range": util.MapStr{
@ -360,8 +361,8 @@ func (engine *Engine) GenerateRawFilter(rule *alerting.Rule, filterParam *alerti
var err error var err error
if rule.Resource.RawFilter != nil { if rule.Resource.RawFilter != nil {
query = util.DeepCopy(rule.Resource.RawFilter).(map[string]interface{}) query = util.DeepCopy(rule.Resource.RawFilter).(map[string]interface{})
}else{ } else {
if !rule.Resource.Filter.IsEmpty(){ if !rule.Resource.Filter.IsEmpty() {
query, err = engine.ConvertFilterQueryToDsl(&rule.Resource.Filter) query, err = engine.ConvertFilterQueryToDsl(&rule.Resource.Filter)
if err != nil { if err != nil {
return nil, err return nil, err
@ -405,7 +406,7 @@ func (engine *Engine) GenerateRawFilter(rule *alerting.Rule, filterParam *alerti
return query, nil return query, nil
} }
func (engine *Engine) ExecuteQuery(rule *alerting.Rule, filterParam *alerting.FilterParam)(*alerting.QueryResult, error){ func (engine *Engine) ExecuteQuery(rule *alerting.Rule, filterParam *alerting.FilterParam) (*alerting.QueryResult, error) {
esClient := elastic.GetClient(rule.Resource.ID) esClient := elastic.GetClient(rule.Resource.ID)
queryResult := &alerting.QueryResult{} queryResult := &alerting.QueryResult{}
indexName := strings.Join(rule.Resource.Objects, ",") indexName := strings.Join(rule.Resource.Objects, ",")
@ -470,7 +471,7 @@ func (engine *Engine) ExecuteQuery(rule *alerting.Rule, filterParam *alerting.Fi
queryResult.MetricData = metricData queryResult.MetricData = metricData
return queryResult, nil return queryResult, nil
} }
func (engine *Engine) GetTargetMetricData(rule *alerting.Rule, isFilterNaN bool, filterParam *alerting.FilterParam)([]alerting.MetricData, *alerting.QueryResult, error){ func (engine *Engine) GetTargetMetricData(rule *alerting.Rule, isFilterNaN bool, filterParam *alerting.FilterParam) ([]alerting.MetricData, *alerting.QueryResult, error) {
queryResult, err := engine.ExecuteQuery(rule, filterParam) queryResult, err := engine.ExecuteQuery(rule, filterParam)
if err != nil { if err != nil {
return nil, queryResult, err return nil, queryResult, err
@ -525,7 +526,7 @@ func (engine *Engine) GetTargetMetricData(rule *alerting.Rule, isFilterNaN bool,
return nil, queryResult, err return nil, queryResult, err
} }
if r, ok := result.(float64); ok { if r, ok := result.(float64); ok {
if math.IsNaN(r) || math.IsInf(r, 0 ){ if math.IsNaN(r) || math.IsInf(r, 0) {
if !isFilterNaN { if !isFilterNaN {
targetData.Data["result"] = append(targetData.Data["result"], []interface{}{timestamp, math.NaN()}) targetData.Data["result"] = append(targetData.Data["result"], []interface{}{timestamp, math.NaN()})
} }
@ -540,10 +541,11 @@ func (engine *Engine) GetTargetMetricData(rule *alerting.Rule, isFilterNaN bool,
} }
return targetMetricData, queryResult, nil return targetMetricData, queryResult, nil
} }
//CheckCondition check whether rule conditions triggered or not
//if triggered returns an ConditionResult // CheckCondition check whether rule conditions triggered or not
//sort conditions by priority desc before check , and then if condition is true, then continue check another group // if triggered returns an ConditionResult
func (engine *Engine) CheckCondition(rule *alerting.Rule)(*alerting.ConditionResult, error){ // sort conditions by priority desc before check , and then if condition is true, then continue check another group
func (engine *Engine) CheckCondition(rule *alerting.Rule) (*alerting.ConditionResult, error) {
var resultItems []alerting.ConditionResultItem var resultItems []alerting.ConditionResultItem
targetMetricData, queryResult, err := engine.GetTargetMetricData(rule, true, nil) targetMetricData, queryResult, err := engine.GetTargetMetricData(rule, true, nil)
conditionResult := &alerting.ConditionResult{ conditionResult := &alerting.ConditionResult{
@ -581,7 +583,7 @@ func (engine *Engine) CheckCondition(rule *alerting.Rule)(*alerting.ConditionRes
continue continue
} }
if r, ok := targetData.Data[dataKey][i][1].(float64); ok { if r, ok := targetData.Data[dataKey][i][1].(float64); ok {
if math.IsNaN(r){ if math.IsNaN(r) {
continue continue
} }
} }
@ -593,7 +595,7 @@ func (engine *Engine) CheckCondition(rule *alerting.Rule)(*alerting.ConditionRes
} }
if evaluateResult == true { if evaluateResult == true {
triggerCount += 1 triggerCount += 1
}else { } else {
triggerCount = 0 triggerCount = 0
} }
if triggerCount >= cond.MinimumPeriodMatch { if triggerCount >= cond.MinimumPeriodMatch {
@ -605,7 +607,7 @@ func (engine *Engine) CheckCondition(rule *alerting.Rule)(*alerting.ConditionRes
IssueTimestamp: targetData.Data[dataKey][i][0], IssueTimestamp: targetData.Data[dataKey][i][0],
RelationValues: map[string]interface{}{}, RelationValues: map[string]interface{}{},
} }
for _, metric := range rule.Metrics.Items{ for _, metric := range rule.Metrics.Items {
resultItem.RelationValues[metric.Name] = queryResult.MetricData[idx].Data[metric.Name][i][1] resultItem.RelationValues[metric.Name] = queryResult.MetricData[idx].Data[metric.Name][i][1]
} }
resultItems = append(resultItems, resultItem) resultItems = append(resultItems, resultItem)
@ -643,10 +645,10 @@ func (engine *Engine) Do(rule *alerting.Rule) error {
} }
} }
if alertItem != nil { if alertItem != nil {
if err != nil{ if err != nil {
alertItem.State = alerting.AlertStateError alertItem.State = alerting.AlertStateError
alertItem.Error = err.Error() alertItem.Error = err.Error()
}else { } else {
for _, actionResult := range alertItem.ActionExecutionResults { for _, actionResult := range alertItem.ActionExecutionResults {
if actionResult.Error != "" { if actionResult.Error != "" {
alertItem.Error = actionResult.Error alertItem.Error = actionResult.Error
@ -687,7 +689,7 @@ func (engine *Engine) Do(rule *alerting.Rule) error {
if err != nil { if err != nil {
return err return err
} }
alertMessage, err := getLastAlertMessage(rule.ID, 2 * time.Minute) alertMessage, err := getLastAlertMessage(rule.ID, 2*time.Minute)
if err != nil { if err != nil {
return fmt.Errorf("get alert message error: %w", err) return fmt.Errorf("get alert message error: %w", err)
} }
@ -797,18 +799,18 @@ func (engine *Engine) Do(rule *alerting.Rule) error {
if err != nil { if err != nil {
return fmt.Errorf("failed to create notification, err: %w", err) return fmt.Errorf("failed to create notification, err: %w", err)
} }
}else{ } else {
alertMessage.Title = alertItem.Title alertMessage.Title = alertItem.Title
alertMessage.Message = alertItem.Message alertMessage.Message = alertItem.Message
alertMessage.ResourceID = rule.Resource.ID alertMessage.ResourceID = rule.Resource.ID
alertMessage.ResourceName= rule.Resource.Name alertMessage.ResourceName = rule.Resource.Name
alertMessage.Priority = priority alertMessage.Priority = priority
err = saveAlertMessage(alertMessage) err = saveAlertMessage(alertMessage)
if err != nil { if err != nil {
return fmt.Errorf("save alert message error: %w", err) return fmt.Errorf("save alert message error: %w", err)
} }
} }
log.Debugf("check condition result of rule %s is %v", conditionResults, rule.ID ) log.Debugf("check condition result of rule %s is %v", conditionResults, rule.ID)
// if alert message status equals ignored , then skip sending message to channel // if alert message status equals ignored , then skip sending message to channel
if alertMessage != nil && alertMessage.Status == alerting.MessageStateIgnored { if alertMessage != nil && alertMessage.Status == alerting.MessageStateIgnored {
@ -834,7 +836,7 @@ func (engine *Engine) Do(rule *alerting.Rule) error {
if err != nil { if err != nil {
return fmt.Errorf("get last notification time from kv error: %w", err) return fmt.Errorf("get last notification time from kv error: %w", err)
} }
if !tm.IsZero(){ if !tm.IsZero() {
rule.LastNotificationTime = tm rule.LastNotificationTime = tm
} }
} }
@ -874,12 +876,12 @@ func (engine *Engine) Do(rule *alerting.Rule) error {
rule.LastTermStartTime = alertMessage.Created rule.LastTermStartTime = alertMessage.Created
} }
if time.Now().Sub(rule.LastTermStartTime.Local()) > throttlePeriod { if time.Now().Sub(rule.LastTermStartTime.Local()) > throttlePeriod {
if rule.LastEscalationTime.IsZero(){ if rule.LastEscalationTime.IsZero() {
tm, err := readTimeFromKV(alerting2.KVLastEscalationTime, []byte(rule.ID)) tm, err := readTimeFromKV(alerting2.KVLastEscalationTime, []byte(rule.ID))
if err != nil { if err != nil {
return fmt.Errorf("get last escalation time from kv error: %w", err) return fmt.Errorf("get last escalation time from kv error: %w", err)
} }
if !tm.IsZero(){ if !tm.IsZero() {
rule.LastEscalationTime = tm rule.LastEscalationTime = tm
} }
} }
@ -899,7 +901,7 @@ func (engine *Engine) Do(rule *alerting.Rule) error {
return nil return nil
} }
func attachTitleMessageToCtx(title, message string, paramsCtx map[string]interface{}) error{ func attachTitleMessageToCtx(title, message string, paramsCtx map[string]interface{}) error {
var ( var (
tplBytes []byte tplBytes []byte
err error err error
@ -917,7 +919,7 @@ func attachTitleMessageToCtx(title, message string, paramsCtx map[string]interfa
return nil return nil
} }
func newParameterCtx(rule *alerting.Rule, checkResults *alerting.ConditionResult, extraParams map[string]interface{} ) map[string]interface{}{ func newParameterCtx(rule *alerting.Rule, checkResults *alerting.ConditionResult, extraParams map[string]interface{}) map[string]interface{} {
var ( var (
conditionParams []util.MapStr conditionParams []util.MapStr
firstGroupValue string firstGroupValue string
@ -972,10 +974,10 @@ func newParameterCtx(rule *alerting.Rule, checkResults *alerting.ConditionResult
max = checkResults.QueryResult.Max max = checkResults.QueryResult.Max
if v, ok := min.(int64); ok { if v, ok := min.(int64); ok {
//expand 60s //expand 60s
min = time.UnixMilli(v).Add(-time.Second*60).UTC().Format("2006-01-02T15:04:05.999Z") min = time.UnixMilli(v).Add(-time.Second * 60).UTC().Format("2006-01-02T15:04:05.999Z")
} }
if v, ok := max.(int64); ok { if v, ok := max.(int64); ok {
max = time.UnixMilli(v).Add(time.Second*60).UTC().Format("2006-01-02T15:04:05.999Z") max = time.UnixMilli(v).Add(time.Second * 60).UTC().Format("2006-01-02T15:04:05.999Z")
} }
} }
paramsCtx := util.MapStr{ paramsCtx := util.MapStr{
@ -1004,7 +1006,7 @@ func (engine *Engine) Test(rule *alerting.Rule, msgType string) ([]alerting.Acti
if err != nil { if err != nil {
return nil, fmt.Errorf("check condition error:%w", err) return nil, fmt.Errorf("check condition error:%w", err)
} }
alertMessage, err := getLastAlertMessage(rule.ID, 2 * time.Minute) alertMessage, err := getLastAlertMessage(rule.ID, 2*time.Minute)
if err != nil { if err != nil {
return nil, fmt.Errorf("get alert message error: %w", err) return nil, fmt.Errorf("get alert message error: %w", err)
} }
@ -1015,19 +1017,19 @@ func (engine *Engine) Test(rule *alerting.Rule, msgType string) ([]alerting.Acti
if alertMessage != nil { if alertMessage != nil {
triggerAt = alertMessage.Created triggerAt = alertMessage.Created
} }
paramsCtx := newParameterCtx(rule, checkResults,util.MapStr{ paramsCtx := newParameterCtx(rule, checkResults, util.MapStr{
alerting2.ParamEventID: util.GetUUID(), alerting2.ParamEventID: util.GetUUID(),
alerting2.ParamTimestamp: now.Unix(), alerting2.ParamTimestamp: now.Unix(),
"duration": now.Sub(triggerAt).String(), "duration": now.Sub(triggerAt).String(),
"trigger_at": triggerAt.Unix(), "trigger_at": triggerAt.Unix(),
} ) })
if msgType == "escalation" || msgType == "notification" { if msgType == "escalation" || msgType == "notification" {
title, message := rule.GetNotificationTitleAndMessage() title, message := rule.GetNotificationTitleAndMessage()
err = attachTitleMessageToCtx(title, message, paramsCtx) err = attachTitleMessageToCtx(title, message, paramsCtx)
if err != nil { if err != nil {
return nil, err return nil, err
} }
}else if msgType == "recover_notification" { } else if msgType == "recover_notification" {
if rule.RecoveryNotificationConfig == nil { if rule.RecoveryNotificationConfig == nil {
return nil, fmt.Errorf("recovery notification must not be empty") return nil, fmt.Errorf("recovery notification must not be empty")
} }
@ -1035,7 +1037,7 @@ func (engine *Engine) Test(rule *alerting.Rule, msgType string) ([]alerting.Acti
if err != nil { if err != nil {
return nil, err return nil, err
} }
}else{ } else {
return nil, fmt.Errorf("unkonwn parameter msg type") return nil, fmt.Errorf("unkonwn parameter msg type")
} }
@ -1060,7 +1062,7 @@ func (engine *Engine) Test(rule *alerting.Rule, msgType string) ([]alerting.Acti
} }
if len(channels) > 0 { if len(channels) > 0 {
actionResults, _ = performChannels(channels, paramsCtx, true) actionResults, _ = performChannels(channels, paramsCtx, true)
}else{ } else {
return nil, fmt.Errorf("no useable channel") return nil, fmt.Errorf("no useable channel")
} }
return actionResults, nil return actionResults, nil
@ -1080,7 +1082,7 @@ func performChannels(channels []alerting.Channel, ctx map[string]interface{}, ra
log.Error(err) log.Error(err)
errCount++ errCount++
errStr = err.Error() errStr = err.Error()
}else{ } else {
if !channel.Enabled { if !channel.Enabled {
continue continue
} }
@ -1094,7 +1096,7 @@ func performChannels(channels []alerting.Channel, ctx map[string]interface{}, ra
Result: string(resBytes), Result: string(resBytes),
Error: errStr, Error: errStr,
Message: string(messageBytes), Message: string(messageBytes),
ExecutionTime: int(time.Now().UnixNano()/1e6), ExecutionTime: int(time.Now().UnixNano() / 1e6),
ChannelType: channel.SubType, ChannelType: channel.SubType,
ChannelName: channel.Name, ChannelName: channel.Name,
ChannelID: channel.ID, ChannelID: channel.ID,
@ -1103,8 +1105,6 @@ func performChannels(channels []alerting.Channel, ctx map[string]interface{}, ra
return actionResults, errCount return actionResults, errCount
} }
func (engine *Engine) GenerateTask(rule alerting.Rule) func(ctx context.Context) { func (engine *Engine) GenerateTask(rule alerting.Rule) func(ctx context.Context) {
return func(ctx context.Context) { return func(ctx context.Context) {
defer func() { defer func() {
@ -1120,17 +1120,17 @@ func (engine *Engine) GenerateTask(rule alerting.Rule) func(ctx context.Context)
} }
} }
func CollectMetricData(agg interface{}, groupValues string, metricData *[]alerting.MetricData){ func CollectMetricData(agg interface{}, groupValues string, metricData *[]alerting.MetricData) {
if aggM, ok := agg.(map[string]interface{}); ok { if aggM, ok := agg.(map[string]interface{}); ok {
if targetAgg, ok := aggM["filter_agg"]; ok { if targetAgg, ok := aggM["filter_agg"]; ok {
collectMetricData(targetAgg, groupValues, metricData) collectMetricData(targetAgg, groupValues, metricData)
}else{ } else {
collectMetricData(aggM, groupValues, metricData) collectMetricData(aggM, groupValues, metricData)
} }
} }
} }
func collectMetricData(agg interface{}, groupValues string, metricData *[]alerting.MetricData){ func collectMetricData(agg interface{}, groupValues string, metricData *[]alerting.MetricData) {
if aggM, ok := agg.(map[string]interface{}); ok { if aggM, ok := agg.(map[string]interface{}); ok {
if timeBks, ok := aggM["time_buckets"].(map[string]interface{}); ok { if timeBks, ok := aggM["time_buckets"].(map[string]interface{}); ok {
if bks, ok := timeBks["buckets"].([]interface{}); ok { if bks, ok := timeBks["buckets"].([]interface{}); ok {
@ -1139,10 +1139,10 @@ func collectMetricData(agg interface{}, groupValues string, metricData *[]alerti
GroupValues: strings.Split(groupValues, "*"), GroupValues: strings.Split(groupValues, "*"),
} }
for _, bk := range bks { for _, bk := range bks {
if bkM, ok := bk.(map[string]interface{}); ok{ if bkM, ok := bk.(map[string]interface{}); ok {
for k, v := range bkM { for k, v := range bkM {
if k == "key" || k == "key_as_string" || k== "doc_count"{ if k == "key" || k == "key_as_string" || k == "doc_count" {
continue continue
} }
if len(k) > 5 { //just store a,b,c if len(k) > 5 { //just store a,b,c
@ -1151,7 +1151,7 @@ func collectMetricData(agg interface{}, groupValues string, metricData *[]alerti
if vm, ok := v.(map[string]interface{}); ok { if vm, ok := v.(map[string]interface{}); ok {
if metricVal, ok := vm["value"]; ok { if metricVal, ok := vm["value"]; ok {
md.Data[k] = append(md.Data[k], alerting.TimeMetricData{bkM["key"], metricVal}) md.Data[k] = append(md.Data[k], alerting.TimeMetricData{bkM["key"], metricVal})
}else{ } else {
//percentiles agg type //percentiles agg type
switch vm["values"].(type) { switch vm["values"].(type) {
case []interface{}: case []interface{}:
@ -1176,12 +1176,12 @@ func collectMetricData(agg interface{}, groupValues string, metricData *[]alerti
} }
} }
*metricData = append(*metricData,md) *metricData = append(*metricData, md)
} }
}else{ } else {
for k, v := range aggM { for k, v := range aggM {
if k == "key" || k== "doc_count"{ if k == "key" || k == "doc_count" {
continue continue
} }
if vm, ok := v.(map[string]interface{}); ok { if vm, ok := v.(map[string]interface{}); ok {
@ -1227,7 +1227,7 @@ func getLastAlertMessageFromES(ruleID string) (*alerting.AlertMessage, error) {
q := orm.Query{ q := orm.Query{
RawQuery: util.MustToJSONBytes(queryDsl), RawQuery: util.MustToJSONBytes(queryDsl),
} }
err, searchResult := orm.Search(alerting.AlertMessage{}, &q ) err, searchResult := orm.Search(alerting.AlertMessage{}, &q)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -1240,7 +1240,7 @@ func getLastAlertMessageFromES(ruleID string) (*alerting.AlertMessage, error) {
return message, err return message, err
} }
func getLastAlertMessage(ruleID string, duration time.Duration) (*alerting.AlertMessage, error ){ func getLastAlertMessage(ruleID string, duration time.Duration) (*alerting.AlertMessage, error) {
messageBytes, err := kv.GetValue(alerting2.KVLastMessageState, []byte(ruleID)) messageBytes, err := kv.GetValue(alerting2.KVLastMessageState, []byte(ruleID))
if err != nil { if err != nil {
return nil, err return nil, err
@ -1280,15 +1280,14 @@ func saveAlertMessage(message *alerting.AlertMessage) error {
return err return err
} }
func readTimeFromKV(bucketKey string, key []byte) (time.Time, error) {
func readTimeFromKV(bucketKey string, key []byte)(time.Time, error){
timeBytes, err := kv.GetValue(bucketKey, key) timeBytes, err := kv.GetValue(bucketKey, key)
zeroTime := time.Time{} zeroTime := time.Time{}
if err != nil { if err != nil {
return zeroTime, err return zeroTime, err
} }
timeStr := string(timeBytes) timeStr := string(timeBytes)
if timeStr != ""{ if timeStr != "" {
return time.ParseInLocation(time.RFC3339, string(timeBytes), time.UTC) return time.ParseInLocation(time.RFC3339, string(timeBytes), time.UTC)
} }
return zeroTime, nil return zeroTime, nil

2
service/alerting/elasticsearch/engine_test.go
View File

@ -40,7 +40,7 @@ import (
"time" "time"
) )
func TestEngine( t *testing.T) { func TestEngine(t *testing.T) {
rule := alerting.Rule{ rule := alerting.Rule{
ID: util.GetUUID(), ID: util.GetUUID(),
Created: time.Now(), Created: time.Now(),

2
service/alerting/elasticsearch/init.go
View File

@ -29,7 +29,7 @@ package elasticsearch
import "infini.sh/console/service/alerting" import "infini.sh/console/service/alerting"
func init(){ func init() {
eng := Engine{} eng := Engine{}
alerting.RegistEngine("elasticsearch", &eng) alerting.RegistEngine("elasticsearch", &eng)
} }

9
service/alerting/engine.go
View File

@ -36,17 +36,18 @@ import (
type Engine interface { type Engine interface {
GenerateQuery(rule *alerting.Rule, filterParam *alerting.FilterParam) (interface{}, error) GenerateQuery(rule *alerting.Rule, filterParam *alerting.FilterParam) (interface{}, error)
ExecuteQuery(rule *alerting.Rule, filterParam *alerting.FilterParam)(*alerting.QueryResult, error) ExecuteQuery(rule *alerting.Rule, filterParam *alerting.FilterParam) (*alerting.QueryResult, error)
CheckCondition(rule *alerting.Rule)(*alerting.ConditionResult, error) CheckCondition(rule *alerting.Rule) (*alerting.ConditionResult, error)
GenerateTask(rule alerting.Rule) func(ctx context.Context) GenerateTask(rule alerting.Rule) func(ctx context.Context)
Test(rule *alerting.Rule, msgType string) ([]alerting.ActionExecutionResult, error) Test(rule *alerting.Rule, msgType string) ([]alerting.ActionExecutionResult, error)
GetTargetMetricData(rule *alerting.Rule, isFilterNaN bool, filterParam *alerting.FilterParam)([]alerting.MetricData, *alerting.QueryResult, error) GetTargetMetricData(rule *alerting.Rule, isFilterNaN bool, filterParam *alerting.FilterParam) ([]alerting.MetricData, *alerting.QueryResult, error)
} }
var ( var (
alertEngines = map[string] Engine{} alertEngines = map[string]Engine{}
alertEnginesMutex = sync.RWMutex{} alertEnginesMutex = sync.RWMutex{}
) )
func RegistEngine(typ string, engine Engine) { func RegistEngine(typ string, engine Engine) {
alertEnginesMutex.Lock() alertEnginesMutex.Lock()
defer alertEnginesMutex.Unlock() defer alertEnginesMutex.Unlock()

4
service/alerting/env.go
View File

@ -37,7 +37,7 @@ import (
log "src/github.com/cihub/seelog" log "src/github.com/cihub/seelog"
) )
func GetEnvVariables() (map[string]interface{}, error){ func GetEnvVariables() (map[string]interface{}, error) {
configFile := global.Env().GetConfigFile() configFile := global.Env().GetConfigFile()
envVariables, err := config.LoadEnvVariables(configFile) envVariables, err := config.LoadEnvVariables(configFile)
if err != nil { if err != nil {
@ -64,7 +64,7 @@ func GetEnvVariables() (map[string]interface{}, error){
return envVariables, nil return envVariables, nil
} }
func GetInnerConsoleEndpoint() (string, error){ func GetInnerConsoleEndpoint() (string, error) {
appConfig := &config2.AppConfig{ appConfig := &config2.AppConfig{
UI: config2.UIConfig{}, UI: config2.UIConfig{},
} }

6
service/alerting/funcs/date.go
View File

@ -32,10 +32,10 @@ import (
"time" "time"
) )
func datetimeInZone(zone string, date interface{}) string{ func datetimeInZone(zone string, date interface{}) string {
return _dateInZone("2006-01-02 15:04:05", date, zone) return _dateInZone("2006-01-02 15:04:05", date, zone)
} }
func datetime(date interface{}) string{ func datetime(date interface{}) string {
return _dateInZone("2006-01-02 15:04:05", date, "Local") return _dateInZone("2006-01-02 15:04:05", date, "Local")
} }
@ -58,7 +58,7 @@ func _dateInZone(fmt string, date interface{}, zone string) string {
t = *date t = *date
case int64: case int64:
if date > 1e12 { if date > 1e12 {
date = date/1000 date = date / 1000
} }
t = time.Unix(date, 0) t = time.Unix(date, 0)
case int: case int:

8
service/alerting/funcs/elastic.go
View File

@ -35,7 +35,7 @@ import (
"strings" "strings"
) )
func lookup(directory string, id string) interface{}{ func lookup(directory string, id string) interface{} {
directory = strings.TrimSpace(directory) directory = strings.TrimSpace(directory)
if directory == "" { if directory == "" {
return "empty_directory" return "empty_directory"
@ -46,8 +46,8 @@ func lookup(directory string, id string) interface{}{
kv := strings.Split(part, "=") kv := strings.Split(part, "=")
if len(kv) == 2 { if len(kv) == 2 {
k := strings.TrimSpace(kv[0]) k := strings.TrimSpace(kv[0])
kvs[k]= strings.TrimSpace(kv[1]) kvs[k] = strings.TrimSpace(kv[1])
}else{ } else {
log.Debugf("got unexpected directory part: %s", part) log.Debugf("got unexpected directory part: %s", part)
} }
} }
@ -59,7 +59,7 @@ func lookup(directory string, id string) interface{}{
return kvs["default"] return kvs["default"]
} }
func lookupMetadata(object string, property string, defaultValue string, id string) interface{}{ func lookupMetadata(object string, property string, defaultValue string, id string) interface{} {
var ( var (
cfgM = util.MapStr{} cfgM = util.MapStr{}
buf []byte buf []byte

4
service/alerting/funcs/numberic.go
View File

@ -35,10 +35,10 @@ import (
"strconv" "strconv"
) )
func toFixed(precision int, num float64) float64{ func toFixed(precision int, num float64) float64 {
return util.ToFixed(num, precision) return util.ToFixed(num, precision)
} }
func add(a, b interface{}) float64{ func add(a, b interface{}) float64 {
av := ToFloat64(a) av := ToFloat64(a)
bv := ToFloat64(b) bv := ToFloat64(b)
return av + bv return av + bv

2
service/alerting/funcs/strings.go
View File

@ -37,7 +37,7 @@ import (
func substring(start, end int, s string) string { func substring(start, end int, s string) string {
runes := []rune(s) runes := []rune(s)
length := len(runes) length := len(runes)
if start < 0 || start > length || end < 0 || end > length{ if start < 0 || start > length || end < 0 || end > length {
return s return s
} }
return string(runes[start:end]) return string(runes[start:end])