From f4e6d4e45e90d8ed4157c462f908cbd1c66f37fc Mon Sep 17 00:00:00 2001 From: silenceqi Date: Thu, 10 Dec 2020 21:28:19 +0800 Subject: [PATCH] add thread chart and modify cluster-moniter chart style --- web/mock/dashboard/cluster_monitor.js | 280 ++++++++++++++++++---- web/package.json | 15 +- web/src/pages/Dashboard/ClusterMonitor.js | 225 ++++++++++------- 3 files changed, 389 insertions(+), 131 deletions(-) diff --git a/web/mock/dashboard/cluster_monitor.js b/web/mock/dashboard/cluster_monitor.js index ec2b7ac0..0a031aac 100644 --- a/web/mock/dashboard/cluster_monitor.js +++ b/web/mock/dashboard/cluster_monitor.js @@ -1,4 +1,4 @@ -import { func } from "prop-types"; +import fetch from 'node-fetch'; let data = JSON.parse(`{ "cluster_stats" : { @@ -478,54 +478,246 @@ let data = JSON.parse(`{ } ]`); +const apiUrls = { + CLUSTER_OVERVIEW: { + path:'/.monitoring-es-*/_search', + body: `{ + "_source": [ "cluster_stats"], + "size": 1, + "sort": [ + { + "timestamp": { + "order": "desc" + } + } + ], + "query": { + "bool": { + "must": [ + { + "match": { + "type": "cluster_stats" + } + } + ], + "filter": [ + { + "range": { + "timestamp": { + "gte": "now-1h", + "lte": "now" + } + } + } + ] + } + } + }` + }, + "GET_ES_NODE_STATS":{ + path: '/.monitoring-es-*/_search', + body: `{ + "size": 0, + "query": { + "bool": { + "must": [ + { + "match": { + "type": "node_stats" + } + } + ], + "filter": [ + { + "range": { + "timestamp": { + "gte": "now-1h", + "lte": "now" + } + } + } + ] + } + }, + "aggs": { + "nodes": { + "terms": { + "field": "source_node.name", + "size": 10 + }, + "aggs": { + "metrics": { + "date_histogram": { + "field": "timestamp", + "fixed_interval": "30s" + }, + "aggs": { + "cpu_used": { + "max": { + "field": "node_stats.process.cpu.percent" + } + }, + "heap_used": { + "max": { + "field": "node_stats.jvm.mem.heap_used_in_bytes" + } + }, + "heap_percent": { + "max": { + "field": "node_stats.jvm.mem.heap_used_percent" + } + }, + "search_query_total": { + "max": { + "field": "node_stats.indices.search.query_total" + } + }, + "search_query_time": { + "max": { + "field": "node_stats.indices.search.query_time_in_millis" + } + }, + "ds": { + "derivative": { + "buckets_path": "search_query_total" + } + }, + "ds1": { + "derivative": { + "buckets_path": "search_query_time" + } + }, + "index_total": { + "max": { + "field": "node_stats.indices.indexing.index_total" + } + }, + "index_time": { + "max": { + "field": "node_stats.indices.indexing.index_time_in_millis" + } + }, + "ds3": { + "derivative": { + "buckets_path": "index_total" + } + }, + "ds4": { + "derivative": { + "buckets_path": "index_time" + } + }, + "read_threads_queue":{ + "max": { + "field": "node_stats.thread_pool.get.queue" + } + }, + "write_threads_queue":{ + "max": { + "field": "node_stats.thread_pool.write.queue" + } + } + } + } + } + } + } + }` + } +}; + + const gatewayUrl = 'http://localhost:8001'; + +function getClusterOverview(){ + return fetch(gatewayUrl+apiUrls.CLUSTER_OVERVIEW.path, { + method: 'POST', + body: apiUrls.CLUSTER_OVERVIEW.body, + headers:{ + 'Content-Type': 'application/json' + } + }).then(esRes=>{ + return esRes.json(); + }).then(rel=>{ + if(rel.hits.hits.length>0){ + var rdata = rel.hits.hits[0]._source; + }else{ + rdata = data; + } + let cluster_stats = rdata.cluster_stats; + let result = { + elasticsearch:{ + cluster_stats:{ + status: cluster_stats.status, + indices: { + count: cluster_stats.indices.count, + docs: cluster_stats.indices.docs, + shards: cluster_stats.indices.shards, + store: cluster_stats.indices.store, + }, + nodes: { + count:{ + total: cluster_stats.nodes.count.total, + }, + fs: cluster_stats.nodes.fs, + jvm: { + max_uptime_in_millis: cluster_stats.nodes.jvm.max_uptime_in_millis, + mem: cluster_stats.nodes.jvm.mem, + } + } + } + } + }; + return result; + }); +} + +function getNodesStats(){ + return fetch(gatewayUrl+apiUrls.GET_ES_NODE_STATS.path, { + method: 'POST', + body: apiUrls.GET_ES_NODE_STATS.body, + headers:{ + 'Content-Type': 'application/json' + } + }).then(esRes=>{ + return esRes.json(); + }).then(rel=>{ + //console.log(rel); + if(rel.aggregations.nodes.buckets.length>0){ + var rdata = rel.aggregations.nodes.buckets; + //console.log(rdata); + }else{ + rdata = nodesStats; + } + return rdata; + }); +} + export default { 'GET /dashboard/cluster/overview': function(req, res){ - let cluster_stats = data.cluster_stats; - let result = { - elasticsearch:{ - cluster_stats:{ - status: cluster_stats.status, - indices: { - count: cluster_stats.indices.count, - docs: cluster_stats.indices.docs, - shards: cluster_stats.indices.shards, - store: cluster_stats.indices.store, - }, - nodes: { - count:{ - total: cluster_stats.nodes.count.total, - }, - fs: cluster_stats.nodes.fs, - jvm: { - max_uptime_in_millis: cluster_stats.nodes.jvm.max_uptime_in_millis, - mem: cluster_stats.nodes.jvm.mem, - } - } - } - } - }; - res.send(result); + //console.log(typeof fetch); + getClusterOverview().then((result)=>{ + //console.log(result); + res.send(result); + }).catch(err=>{ + console.log(err); + }); }, 'GET /dashboard/cluster/nodes_stats': function(req, res) { - // var statsData = [{ - // node_name: nodesStats.source_node.name, - // node_id: nodesStats.source_node.uuid, - // timestamp: nodesStats.timestamp, - // node_stats: { - // jvm: nodesStats.node_stats.jvm, - // process: nodesStats.node_stats.process, - // node_master: nodesStats.node_stats.node_master, - // indices: { - // search: nodesStats.node_stats.indices.search, - // indexing: nodesStats.node_stats.indices.indices, - // }, - // fs: nodesStats.node_stats.fs, - // os: nodesStats.node_stats.os, - // } - // }]; - + Promise.all([ getNodesStats()]).then((values) => { + //console.log(values); res.send({ - nodes_stats: nodesStats + // elasticsearch: values[0].elasticsearch, + nodes_stats: values[0], }); + }).catch(err=>{ + console.log(err); + }); + // getNodesStats().then((rdata)=>{ + // res.send({ + // nodes_stats: rdata + // }); + // }).catch(err=>{ + // console.log(err); + // }) }, }; \ No newline at end of file diff --git a/web/package.json b/web/package.json index 933f9ce8..7fa30ccc 100644 --- a/web/package.json +++ b/web/package.json @@ -45,17 +45,22 @@ "repl": "^0.1.3" }, "devDependencies": { + "antd-pro-merge-less": "^0.1.0", "antd-theme-webpack-plugin": "^1.1.8", "autod": "^3.0.1", "autod-egg": "^1.1.0", - "antd-pro-merge-less": "^0.1.0", "babel-eslint": "^8.1.2", "babel-plugin-dva-hmr": "^0.4.1", "babel-plugin-import": "^1.6.3", "babel-plugin-transform-decorators-legacy": "^1.3.4", + "egg": "^2.4.1", "egg-bin": "^4.3.7", "egg-ci": "^1.8.0", "egg-mock": "^3.15.0", + "egg-scripts": "^2.5.1", + "egg-validate": "^2.0.2", + "egg-view-assets": "^1.6.1", + "egg-view-nunjucks": "^2.2.0", "enzyme": "^3.9.0", "eslint": "^4.18.2", "eslint-config-airbnb": "^17.0.0", @@ -68,15 +73,11 @@ "eslint-plugin-markdown": "^1.0.0-beta.6", "eslint-plugin-react": "^7.11.1", "mockjs": "^1.0.1-beta3", + "node-fetch": "^2.6.1", "redbox-react": "^1.5.0", "umi": "^2.1.2", "umi-plugin-ga": "^1.1.3", - "umi-plugin-react": "^1.1.1", - "egg": "^2.4.1", - "egg-scripts": "^2.5.1", - "egg-validate": "^2.0.2", - "egg-view-assets": "^1.6.1", - "egg-view-nunjucks": "^2.2.0" + "umi-plugin-react": "^1.1.1" }, "engines": { "node": ">=8.9.0" diff --git a/web/src/pages/Dashboard/ClusterMonitor.js b/web/src/pages/Dashboard/ClusterMonitor.js index 5ef08d22..7aadd8d4 100644 --- a/web/src/pages/Dashboard/ClusterMonitor.js +++ b/web/src/pages/Dashboard/ClusterMonitor.js @@ -161,7 +161,7 @@ let generateIndexLatencyData = (target)=>{ } - + let charts = []; class SliderChart extends React.Component { constructor() { super(); @@ -198,11 +198,29 @@ let generateIndexLatencyData = (target)=>{ let pos = `${xname}*${yname}`; return ( +
{ + charts.push(c); + }} + onPlotMove={ev=>{ + charts.forEach((chart)=>{ + chart.showTooltip({ + x: ev.x, + y: ev.y + }); + }); + }} + onPlotLeave={ev=>{ + charts.forEach(chart=>{ + chart.hideTooltip(); + }); + }} scale={scale} - height={300} + height={180} forceFit + padding="auto" // onGetG2Instance={g2Chart => { // chart = g2Chart; // }} @@ -211,7 +229,7 @@ let generateIndexLatencyData = (target)=>{ {this.props.title} - + { color="type" /> +
); } } const styles ={ mainTitle:{ - fontSize:20, + fontSize:14, color:"black", textAlign:"center" }, subTitle:{ - fontSize:16, + fontSize:12, color:"gray", textAlign:"center" } @@ -284,74 +303,71 @@ class StatsCharts extends PureComponent { indexLatency:[], timeScale: {min: moment().subtract(1, 'h').valueOf(), max: new Date()}, } - componentDidMount(){ + fetchData() { let {dispatch} = this.props; - let me = this; - setInterval(function(){ - dispatch({ - type: 'clusterMonitor/fetchClusterNodeStats', - callback: ({nodes_stats})=> { - // var heapStats = [], cpuStats=[],searchLatency=[], indexLatency=[]; - // for(let st of nodes_stats){ - // var fields = { - // time: st.timestamp, - // type: st.node_name, - // } - // heapStats.push({ - // ...fields, - // heap_ratio: st.node_stats.jvm.mem.heap_used_percent, - // }); - // cpuStats.push({ - // ...fields, - // cpu_ratio: st.node_stats.process.cpu.percent, - // }); - // } - let nodesStats = nodes_stats; - let nodeCpu = [],nodeHeap=[],nodeSearchLatency=[],nodeIndexLatency=[]; - let now = moment(1607085646112); - for(let ns of nodesStats){ - let i = 0; - for(let bk of ns.metrics.buckets){ - let fields = { - time: now.subtract(300-30*i , 's').valueOf(), //bk.key_as_string, - type: ns.key, - }; - i++; - nodeCpu.push({ - ...fields, - cpu_ratio: bk.cpu_used.value, - }); - nodeHeap.push({ - ...fields, - heap_ratio: bk.heap_percent.value, - }); - nodeSearchLatency.push({ - ...fields, - latency: bk.ds1 ? (bk.ds1.value/bk.ds.value).toFixed(2): 0, - }); - nodeIndexLatency.push({ - ...fields, - latency: bk.ds1 ? (bk.ds4.value/bk.ds3.value).toFixed(2): 0, - }); - } - } - me.setState({ - heapStats: nodeHeap, - cpuStats: nodeCpu, - searchLatency: nodeSearchLatency, - indexLatency: nodeIndexLatency, - timeScale: {min: moment().subtract(1, 'h').valueOf(), max: new Date()}, - }); - } - }); - }, 10000); + dispatch({ + type: 'clusterMonitor/fetchClusterNodeStats', + callback: ({nodes_stats})=> { + let nodesStats = nodes_stats; + console.log(nodesStats); + let nodeCpu = [],nodeHeap=[],nodeSearchLatency=[],nodeIndexLatency=[], readThreadQueue=[],writeThreadQueue=[]; + //let now = moment(1607085646112); + for(let ns of nodesStats){ + //let i = 0; + for(let bk of ns.metrics.buckets){ + let fields = { + time: bk.key_as_string, //now.subtract(300-30*i , 's').valueOf(), + type: ns.key, + }; + nodeCpu.push({ + ...fields, + cpu_ratio: bk.cpu_used.value, + }); + nodeHeap.push({ + ...fields, + heap_ratio: bk.heap_percent.value, + }); + nodeSearchLatency.push({ + ...fields, + latency: bk.ds1 ? (bk.ds1.value/bk.ds.value).toFixed(2): 0, + }); + nodeIndexLatency.push({ + ...fields, + latency: bk.ds1 ? (bk.ds4.value/bk.ds3.value).toFixed(2): 0, + }); + readThreadQueue.push({ + ...fields, + queue: bk.read_threads_queue.value, + }); + writeThreadQueue.push({ + ...fields, + queue: bk.write_threads_queue.value, + }); + } + } + this.setState({ + heapStats: nodeHeap, + cpuStats: nodeCpu, + searchLatency: nodeSearchLatency, + indexLatency: nodeIndexLatency, + readThreadQueue: readThreadQueue, + writeThreadQueue: writeThreadQueue, + timeScale: {min: moment().subtract(1, 'h').valueOf(), max: new Date()}, + }); + } + }); + } + componentDidMount(){ + this.fetchData(); + setInterval(() =>{ + this.fetchData(); + }, 30000); } render(){ return (
- - + /> - - - + - - + /> - - -
+ + + + + + + + + + ) } } @@ -478,12 +537,18 @@ class StatsCharts extends PureComponent { class ClusterMonitor extends PureComponent { state={ } - componentDidMount() { + fetchData(){ const { dispatch } = this.props; dispatch({ type: 'clusterMonitor/fetchClusterOverview', }); } + componentDidMount() { + this.fetchData(); + setInterval(()=>{ + this.fetchData(); + }, 30000); + } render() { let vstyle = { @@ -517,7 +582,7 @@ class ClusterMonitor extends PureComponent { - } /> + } />