diff --git a/web/mock/dashboard/calculate_timeseries_interval.js b/web/mock/dashboard/calculate_timeseries_interval.js new file mode 100644 index 00000000..ac8ba2d7 --- /dev/null +++ b/web/mock/dashboard/calculate_timeseries_interval.js @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import moment from 'moment'; + +const d = moment.duration; +const roundingRules = [ + [d(500, 'ms'), d(100, 'ms')], + [d(5, 'second'), d(1, 'second')], + [d(7.5, 'second'), d(5, 'second')], + [d(15, 'second'), d(10, 'second')], + [d(45, 'second'), d(30, 'second')], + [d(3, 'minute'), d(1, 'minute')], + [d(9, 'minute'), d(5, 'minute')], + [d(20, 'minute'), d(10, 'minute')], + [d(45, 'minute'), d(30, 'minute')], + [d(2, 'hour'), d(1, 'hour')], + [d(6, 'hour'), d(3, 'hour')], + [d(24, 'hour'), d(12, 'hour')], + [d(1, 'week'), d(1, 'd')], + [d(3, 'week'), d(1, 'week')], + [d(1, 'year'), d(1, 'month')], + [Infinity, d(1, 'year')], +]; + +function find(rules, check) { + function pick(buckets, duration) { + const target = duration / buckets; + let lastResp; + + for (let i = 0; i < rules.length; i++) { + const rule = rules[i]; + const resp = check(rule[0], rule[1], target); + + if (resp == null) { + if (lastResp) { + return lastResp; + } + break; + } + + lastResp = resp; + } + + // fallback to just a number of milliseconds, ensure ms is >= 1 + const ms = Math.max(Math.floor(target), 1); + return moment.duration(ms, 'ms'); + } + + return function (buckets, duration) { + const interval = pick(buckets, duration); + if (interval) { + return moment.duration(interval._data); + } + }; +} + +const revRoundingRules = roundingRules.slice(0).reverse(); + +/* + * 24 hours: 600 seconds + * 12 hours: 300 seconds + * 4 hours: 60 seconds + * 1 hour: 30 seconds + * 15 minutes: 10 seconds + */ +export const calculateAuto = find(revRoundingRules, (bound, interval, target) => { + if (bound > target) { + return interval; + } +}); + + +export function calculateTimeseriesInterval( + lowerBoundInMsSinceEpoch, + upperBoundInMsSinceEpoch, + minIntervalSeconds +) { + const duration = moment.duration(upperBoundInMsSinceEpoch - lowerBoundInMsSinceEpoch, 'ms'); + + return Math.max(minIntervalSeconds, calculateAuto(100, duration).asSeconds()); +} \ No newline at end of file diff --git a/web/mock/dashboard/cluster_monitor.js b/web/mock/dashboard/cluster_monitor.js index ad8b7c4a..c9ee3e0a 100644 --- a/web/mock/dashboard/cluster_monitor.js +++ b/web/mock/dashboard/cluster_monitor.js @@ -1,4 +1,10 @@ import fetch from 'node-fetch'; +import moment from 'moment'; +import {calculateTimeseriesInterval} from './calculate_timeseries_interval'; +import { promises } from 'dns'; +//import {formatTimestampToDuration} from './format_timestamp_to_duration'; +const minIntervalSeconds = 10; + let data = JSON.parse(`{ "cluster_stats" : { @@ -319,6 +325,20 @@ let data = JSON.parse(`{ }, "search_query_time" : { "value" : 7314.0 + }, + "read_threads_queue" : { + "value" : 0.0 + }, + "write_threads_queue" : { + "value" : 0.0 + }, + "search_qps" : { + "value" : 0.0, + "normalized_value" : 2.1 + }, + "index_qps" : { + "value" : 165.0, + "normalized_value" : 5.9 } }, { @@ -357,6 +377,20 @@ let data = JSON.parse(`{ }, "ds4" : { "value" : 102.0 + }, + "read_threads_queue" : { + "value" : 0.0 + }, + "write_threads_queue" : { + "value" : 0.0 + }, + "search_qps" : { + "value" : 0.0, + "normalized_value" : 1.8 + }, + "index_qps" : { + "value" : 165.0, + "normalized_value" : 5.3 } }, { @@ -395,6 +429,20 @@ let data = JSON.parse(`{ }, "ds4" : { "value" : 144.0 + }, + "read_threads_queue" : { + "value" : 0.0 + }, + "write_threads_queue" : { + "value" : 0.0 + }, + "search_qps" : { + "value" : 0.0, + "normalized_value" : 2.5 + }, + "index_qps" : { + "value" : 165.0, + "normalized_value" : 5.0 } }, { @@ -433,6 +481,20 @@ let data = JSON.parse(`{ }, "ds4" : { "value" : 123.0 + }, + "read_threads_queue" : { + "value" : 0.0 + }, + "write_threads_queue" : { + "value" : 0.0 + }, + "search_qps" : { + "value" : 0.0, + "normalized_value" : 3.0 + }, + "index_qps" : { + "value" : 165.0, + "normalized_value" : 5.8 } }, { @@ -471,6 +533,20 @@ let data = JSON.parse(`{ }, "ds4" : { "value" : 19.0 + }, + "read_threads_queue" : { + "value" : 0.0 + }, + "write_threads_queue" : { + "value" : 0.0 + }, + "search_qps" : { + "value" : 0.0, + "normalized_value" : 0.0 + }, + "index_qps" : { + "value" : 165.0, + "normalized_value" : 5.5 } } ] @@ -478,180 +554,193 @@ let data = JSON.parse(`{ } ]`); + function getOverviewBody(params){ + let body = { + _source: [ "cluster_stats"], + size: 1, + sort: [ + { + timestamp: { + order: "desc" + } + } + ], + query: { + bool: { + must: [ + { + match: { + type: "cluster_stats" + } + } + ], + filter: [ + { + range: { + timestamp: { + "gte": params.timeRange.min, + lte: params.timeRange.max + } + } + } + ] + } + } + }; + return JSON.stringify(body); + } + +function getNodesStatsBody(params){ + let min = moment(params.timeRange.min).valueOf(); + let max = moment(params.timeRange.max).valueOf(); + const bucketSizeInSeconds = calculateTimeseriesInterval(min, max, minIntervalSeconds); + console.log(bucketSizeInSeconds); + let body = { + "size": 0, + "query": { + "bool": { + "must": [ + { + "match": { + "type": "node_stats" + } + } + ], + "filter": [ + { + "range": { + "timestamp": { + "gte": params.timeRange.min, + "lte": params.timeRange.max + } + } + } + ] + } + }, + "aggs": { + "nodes": { + "terms": { + "field": "source_node.name", + "size": 10 + }, + "aggs": { + "metrics": { + "date_histogram": { + "field": "timestamp", + "fixed_interval": bucketSizeInSeconds + 's' + }, + "aggs": { + "cpu_used": { + "max": { + "field": "node_stats.process.cpu.percent" + } + }, + "heap_used": { + "max": { + "field": "node_stats.jvm.mem.heap_used_in_bytes" + } + }, + "heap_percent": { + "max": { + "field": "node_stats.jvm.mem.heap_used_percent" + } + }, + "search_query_total": { + "max": { + "field": "node_stats.indices.search.query_total" + } + }, + "search_query_time": { + "max": { + "field": "node_stats.indices.search.query_time_in_millis" + } + }, + "ds": { + "derivative": { + "buckets_path": "search_query_total" + } + }, + "ds1": { + "derivative": { + "buckets_path": "search_query_time" + } + }, + "index_total": { + "max": { + "field": "node_stats.indices.indexing.index_total" + } + }, + "index_time": { + "max": { + "field": "node_stats.indices.indexing.index_time_in_millis" + } + }, + "ds3": { + "derivative": { + "buckets_path": "index_total" + } + }, + "ds4": { + "derivative": { + "buckets_path": "index_time" + } + }, + "search_qps":{ + "derivative": { + "buckets_path": "search_query_total", + "gap_policy": "skip", + "unit": "1s" + } + }, + "index_qps":{ + "derivative": { + "buckets_path": "index_total", + "gap_policy": "skip", + "unit": "1s" + } + }, + "read_threads_queue":{ + "max": { + "field": "node_stats.thread_pool.get.queue" + } + }, + "write_threads_queue":{ + "max": { + "field": "node_stats.thread_pool.write.queue" + } + } + } + } + } + } + } + }; + return JSON.stringify(body); +} + const apiUrls = { CLUSTER_OVERVIEW: { path:'/.monitoring-es-*/_search', - body: `{ - "_source": [ "cluster_stats"], - "size": 1, - "sort": [ - { - "timestamp": { - "order": "desc" - } - } - ], - "query": { - "bool": { - "must": [ - { - "match": { - "type": "cluster_stats" - } - } - ], - "filter": [ - { - "range": { - "timestamp": { - "gte": "now-1h", - "lte": "now" - } - } - } - ] - } - } - }` }, "GET_ES_NODE_STATS":{ path: '/.monitoring-es-*/_search', - body: `{ - "size": 0, - "query": { - "bool": { - "must": [ - { - "match": { - "type": "node_stats" - } - } - ], - "filter": [ - { - "range": { - "timestamp": { - "gte": "now-1h", - "lte": "now" - } - } - } - ] - } - }, - "aggs": { - "nodes": { - "terms": { - "field": "source_node.name", - "size": 10 - }, - "aggs": { - "metrics": { - "date_histogram": { - "field": "timestamp", - "fixed_interval": "30s" - }, - "aggs": { - "cpu_used": { - "max": { - "field": "node_stats.process.cpu.percent" - } - }, - "heap_used": { - "max": { - "field": "node_stats.jvm.mem.heap_used_in_bytes" - } - }, - "heap_percent": { - "max": { - "field": "node_stats.jvm.mem.heap_used_percent" - } - }, - "search_query_total": { - "max": { - "field": "node_stats.indices.search.query_total" - } - }, - "search_query_time": { - "max": { - "field": "node_stats.indices.search.query_time_in_millis" - } - }, - "ds": { - "derivative": { - "buckets_path": "search_query_total" - } - }, - "ds1": { - "derivative": { - "buckets_path": "search_query_time" - } - }, - "index_total": { - "max": { - "field": "node_stats.indices.indexing.index_total" - } - }, - "index_time": { - "max": { - "field": "node_stats.indices.indexing.index_time_in_millis" - } - }, - "ds3": { - "derivative": { - "buckets_path": "index_total" - } - }, - "ds4": { - "derivative": { - "buckets_path": "index_time" - } - }, - "search_qps":{ - "derivative": { - "buckets_path": "search_query_total", - "gap_policy": "skip", - "unit": "1s" - } - }, - "index_qps":{ - "derivative": { - "buckets_path": "index_total", - "gap_policy": "skip", - "unit": "1s" - } - }, - "read_threads_queue":{ - "max": { - "field": "node_stats.thread_pool.get.queue" - } - }, - "write_threads_queue":{ - "max": { - "field": "node_stats.thread_pool.write.queue" - } - } - } - } - } - } - } - }` } }; - const gatewayUrl = 'http://localhost:8001'; + const gatewayUrl = 'http://localhost:9200'; -function getClusterOverview(){ +function getClusterOverview(params){ return fetch(gatewayUrl+apiUrls.CLUSTER_OVERVIEW.path, { method: 'POST', - body: apiUrls.CLUSTER_OVERVIEW.body, + body: getOverviewBody(params), headers:{ 'Content-Type': 'application/json' } }).then(esRes=>{ return esRes.json(); }).then(rel=>{ + //console.log(rel); if(rel.hits.hits.length>0){ var rdata = rel.hits.hits[0]._source; }else{ @@ -681,19 +770,20 @@ function getClusterOverview(){ } } }; - return result; + return Promise.resolve(result); }); } -function getNodesStats(){ +function getNodesStats(params){ return fetch(gatewayUrl+apiUrls.GET_ES_NODE_STATS.path, { method: 'POST', - body: apiUrls.GET_ES_NODE_STATS.body, + body: getNodesStatsBody(params), headers:{ 'Content-Type': 'application/json' } }).then(esRes=>{ - return esRes.json(); + return esRes.json(); + // return esRes.json(); }).then(rel=>{ //console.log(rel); if(rel.aggregations.nodes.buckets.length>0){ @@ -702,21 +792,41 @@ function getNodesStats(){ }else{ rdata = nodesStats; } - return rdata; + return Promise.resolve(rdata); }); } export default { - 'GET /dashboard/cluster/overview': function(req, res){ - //console.log(typeof fetch); - getClusterOverview().then((result)=>{ - //console.log(result); - res.send(result); - }).catch(err=>{ - console.log(err); + 'POST /dashboard/cluster/overview': function(req, res){ + // console.log(1, req.body); + let params = req.body; + !params.timeRange && (params.timeRange={ + min: 'now-1h', + max: 'now' }); + Promise.all([getClusterOverview(params),getNodesStats(params)]).then(function(values){ + let robj = values[0]; + robj = Object.assign(robj, {nodes_stats: values[1]}); + res.send(robj); + }).catch(function(err){ + console.log(err); + }); + // getClusterOverview(params).then((result)=>{ + // //console.log(result); + // res.send(result); + // }).catch(err=>{ + // console.log(err); + // }); }, 'GET /dashboard/cluster/nodes_stats': function(req, res) { + let min = moment(1607839878669 - 2592000000).valueOf(); + const max = moment(1607839878669).valueOf(); + const bucketSizeInSeconds = calculateTimeseriesInterval(min, max, minIntervalSeconds); + const now = moment(); + const timestamp = moment(now).add(bucketSizeInSeconds, 'seconds'); // clone the `now` object + + //console.log(bucketSizeInSeconds); //, formatTimestampToDuration(timestamp, 'until', now)); + Promise.all([ getNodesStats()]).then((values) => { //console.log(values); res.send({ diff --git a/web/package.json b/web/package.json index 7fa30ccc..a89313eb 100644 --- a/web/package.json +++ b/web/package.json @@ -6,6 +6,7 @@ "@ant-design/charts": "^1.0.4", "@ant-design/icons": "^4.0.0", "@antv/data-set": "^0.9.6", + "@antv/g2-brush": "^0.0.2", "@babel/runtime": "^7.1.2", "antd": "^3.26.18", "antd-table-infinity": "^1.1.6", @@ -73,6 +74,7 @@ "eslint-plugin-markdown": "^1.0.0-beta.6", "eslint-plugin-react": "^7.11.1", "mockjs": "^1.0.1-beta3", + "moment-duration-format": "^2.3.2", "node-fetch": "^2.6.1", "redbox-react": "^1.5.0", "umi": "^2.1.2", diff --git a/web/src/pages/Dashboard/ClusterMonitor.js b/web/src/pages/Dashboard/ClusterMonitor.js index 8bed5887..b48a9ab5 100644 --- a/web/src/pages/Dashboard/ClusterMonitor.js +++ b/web/src/pages/Dashboard/ClusterMonitor.js @@ -1,8 +1,10 @@ import React, { PureComponent,Fragment } from 'react'; import { connect } from 'dva'; import { formatMessage, FormattedMessage } from 'umi/locale'; -import { Row, Col, Card,Statistic,Icon, Divider, Skeleton } from 'antd'; -import moment from 'moment'; +import { Row, Col, Card,Statistic,Icon, Divider, DatePicker, Input, Dropdown, Menu, Button, InputNumber, Select } from 'antd'; +import moment, { relativeTimeRounding } from 'moment'; +import Brush from '@antv/g2-brush'; +const { RangePicker } = DatePicker; import { @@ -13,153 +15,6 @@ import { Tooltip, Legend, } from 'bizcharts'; -import { func } from 'prop-types'; - -let generateHeapData = (target)=>{ - let data = []; - let generator = (initTime) => { - var now = new Date(); - var time = initTime||now.getTime(); - var heap1 = ~~(Math.random() * 500) + 200; - var heap2 = ~~(Math.random() * 300) + 512; - if (data.length >= 120) { - data.shift(); - data.shift(); - } - - data.push({ - time: time, - heap_ratio: (heap1 *100) /1024, - type: "node1" - }); - data.push({ - time: time, - heap_ratio: (heap2 *100)/1024, - type: "node2" - }); - !initTime && target.setState({ - data - }); - }; - let stime = new Date(); - for(let i=120;i>0;i--){ - generator(new Date(stime.valueOf()- i * 1000 * 30)); - } - target.setState({ - data - }); - setInterval(()=>{generator(null)}, 30000); -} - -let generateCpuData = (target)=>{ - let data = []; - let generator = (initTime) => { - var now = new Date(); - var time = initTime || now.getTime(); - var cpu1 = ~~(Math.random()*5) + 0.1; - var cpu2 = ~~(Math.random()*3) +0.2; - if (data.length >= 120) { - data.shift(); - data.shift(); - } - - data.push({ - time: time, - cpu_ratio: cpu1, - type: "node1" - }); - data.push({ - time: time, - cpu_ratio: cpu2, - type: "node2" - }); - !initTime && target.setState({ - data - }); - }; - let stime = new Date(); - for(let i=120;i>0;i--){ - generator(new Date(stime.valueOf()- i * 1000 * 30)); - } - target.setState({ - data - }); - setInterval(()=>{generator(null)}, 30000); -} - -let generateSearchLatencyData = (target)=>{ - let data = []; - let generator = (initTime) => { - var now = new Date(); - var time = initTime || now.getTime(); - var latency1 = ~~(Math.random()*100) + 10; - var latency2 = ~~(Math.random()*150) +30; - if (data.length >= 120) { - data.shift(); - data.shift(); - } - - data.push({ - time: time, - latency: latency1, - type: "node1" - }); - data.push({ - time: time, - latency: latency2, - type: "node2" - }); - !initTime && target.setState({ - data - }); - }; - let stime = new Date(); - for(let i=120;i>0;i--){ - generator(new Date(stime.valueOf()- i * 1000 * 30)); - } - target.setState({ - data - }); - setInterval(()=>{generator(null)}, 30000); -} - -let generateIndexLatencyData = (target)=>{ - let data = []; - let generator = (initTime) => { - var now = new Date(); - var time = initTime || now.getTime(); - var latency1 = ~~(Math.random()*400) + 50; - var latency2 = ~~(Math.random()*500) +20; - if (data.length >= 120) { - data.shift(); - data.shift(); - } - - data.push({ - time: time, - latency: latency1, - type: "node1" - }); - data.push({ - time: time, - latency: latency2, - type: "node2" - }); - !initTime && target.setState({ - data - }); - }; - - let stime = new Date(); - for(let i=120;i>0;i--){ - generator(new Date(stime.valueOf()- i * 1000 * 30)); - } - target.setState({ - data - }); - setInterval(()=>{generator(null)}, 30000); -} - let charts = []; class SliderChart extends React.Component { @@ -201,8 +56,25 @@ let generateIndexLatencyData = (target)=>{