From 355b6d3a9ceb87a2106061cf260a4b3b061a0aa1 Mon Sep 17 00:00:00 2001 From: Kassian Sun Date: Thu, 8 Jun 2023 14:09:37 +0800 Subject: [PATCH 1/7] [migration] don't skip bulk error check if count unmatch --- .../index_migration/index_migration.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/plugin/task_manager/index_migration/index_migration.go b/plugin/task_manager/index_migration/index_migration.go index ec9af750..e53f6bf5 100644 --- a/plugin/task_manager/index_migration/index_migration.go +++ b/plugin/task_manager/index_migration/index_migration.go @@ -461,15 +461,18 @@ func (p *processor) checkBulkPipelineTaskStatus(bulkTask *task.Task, cfg *migrat ) successDocs = migration_util.GetMapIntValue(bulkLabels, "success_docs") - if !cfg.Target.SkipCountCheck && successDocs != totalDocs { - return true, successDocs, fmt.Errorf("bulk complete but docs count unmatch: %d / %d, invalid docs: [%s] (reasons: [%s]), failure docs: [%s] (reasons: [%s])", successDocs, totalDocs, invalidDocs, invalidReasons, failureDocs, failureReasons) - } - - // successDocs matched but has errors - if bulkTask.Status == task.StatusError { - return true, successDocs, nil + if successDocs != totalDocs { + // check count + if !cfg.Target.SkipCountCheck { + return true, successDocs, fmt.Errorf("bulk complete but docs count unmatch: %d / %d, invalid docs: [%s] (reasons: [%s]), failure docs: [%s] (reasons: [%s])", successDocs, totalDocs, invalidDocs, invalidReasons, failureDocs, failureReasons) + } + // has errors + if bulkTask.Status == task.StatusError { + return true, successDocs, fmt.Errorf("bulk pipeline failed") + } } + // successDocs matched, return ok return true, successDocs, nil } From ab018479a3bc9c00861db6b54810959bc2fbed83 Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 8 Jun 2023 14:22:10 +0800 Subject: [PATCH 2/7] add default certs and fix build and tpl --- build.sh | 6 +++++- config/certs/ca.crt | 24 ++++++++++++++++++++++++ config/certs/ca.key | 28 ++++++++++++++++++++++++++++ config/install_agent.tpl | 18 +++++++++--------- console.yml | 26 +++++++++++++------------- 5 files changed, 79 insertions(+), 23 deletions(-) create mode 100644 config/certs/ca.crt create mode 100644 config/certs/ca.key diff --git a/build.sh b/build.sh index 17225641..c178bef6 100644 --- a/build.sh +++ b/build.sh @@ -27,7 +27,10 @@ GOROOT="/infini/go-pkgs/go-loongarch" PATH=$GOROOT/bin:$PATH make build-linux-lo #copy-configs cp -rf $WORKBASE/framework/LICENSE $WORKDIR/bin && cat $WORKBASE/framework/NOTICE $WORKDIR/NOTICE > $WORKDIR/bin/NOTICE -mkdir -p $WORKDIR/bin/config && cp $WORKDIR/config/*.json $WORKDIR/bin/config && cp -rf $WORKDIR/config/*.tpl $WORKDIR/bin/config +mkdir -p $WORKDIR/bin/config +cp $WORKDIR/config/*.json $WORKDIR/bin/config +cp -rf $WORKDIR/config/*.tpl $WORKDIR/bin/config +cp -rf $WORKDIR/config/certs $WORKDIR/bin/config cd $WORKDIR/bin for t in 386 amd64 arm64 armv5 armv6 armv7 loong64 mips mips64 mips64le mipsle riscv64 ; do @@ -50,6 +53,7 @@ WORKDIR \${APP_HOME} COPY ["$PNAME-linux-$t", "$PNAME.yml", "\${APP_HOME}/"] COPY ["config", "\${APP_HOME}/config"] +COPY ["config/certs", "\${APP_HOME}/config/certs"] CMD ["/opt/$PNAME/${PNAME}-linux-$t"] EOF diff --git a/config/certs/ca.crt b/config/certs/ca.crt new file mode 100644 index 00000000..1d83be0b --- /dev/null +++ b/config/certs/ca.crt @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIID+zCCAuOgAwIBAgIUEctl/ds6wIoJGTW3PQc5L/0VlQ4wDQYJKoZIhvcNAQEL +BQAwgYwxCzAJBgNVBAYTAkNOMQ4wDAYDVQQIDAVodW5hbjERMA8GA1UEBwwIY2hh +bmdzaGExEzARBgNVBAoMCmluZmluaWxhYnMxCzAJBgNVBAsMAml0MRcwFQYDVQQD +DA5pbmZpbmlsYWJzLmNvbTEfMB0GA1UEAwwWcmVsZWFzZS5pbmZpbmlsYWJzLmNv +bTAeFw0yMzA2MDgwNTI1MzVaFw0zMzA2MDUwNTI1MzVaMIGMMQswCQYDVQQGEwJD +TjEOMAwGA1UECAwFaHVuYW4xETAPBgNVBAcMCGNoYW5nc2hhMRMwEQYDVQQKDApp +bmZpbmlsYWJzMQswCQYDVQQLDAJpdDEXMBUGA1UEAwwOaW5maW5pbGFicy5jb20x +HzAdBgNVBAMMFnJlbGVhc2UuaW5maW5pbGFicy5jb20wggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQD43Co0NROYEeeZWWH4O3m+V7U+1/4DVuAm+9u1bxqi +OnliE24wm9+gk3HEwdr6pMGTfMWS8BMmqUpjjgFVK4Tcur87Cqjq7XDe8j7h5Ipi +8yVUAgqF4wesIZpGtxXRZwwGWwRu38zX5CAa9n9Xbp0Y7tDdINRk0vLCp7VQbd2N +VbMXgqygJAaAImdNfrddAmojWJ92LCT5HKcDNq8Z62VwtLqOUePiEJxm1sUts9tT +sX8XlyLljz2aoWcX+Gzin8HkOftnpYeHptDL26Q2FyW7TYZR4oFuhU6FQ/YPfCsE +m/sxVC7BIBWal4DF29ZiivWvWk+wBNq03LxR1/TvJtOdAgMBAAGjUzBRMB0GA1Ud +DgQWBBSsTKk1fbAbRxPIydDSatzKh+YaGzAfBgNVHSMEGDAWgBSsTKk1fbAbRxPI +ydDSatzKh+YaGzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAl +0p1/QMgdahq22SY+shvvxH9AbtyQo3XFZjWXd0+rMbt0uci640NDhaZxBeMgDWaJ +jRe+K7sw+AhnIWzH9RbaYQfVAXxjFb7kHSb93bezXjA3m21O5KpwiQyaXCbsIVSf +n59pd3+EV/Q96EclNMoixpDUVtqI7i046/3imZ4XyBDpQPWCajaKpp8rkypCvykK +KQ7BGF8lr3WyAgfsoHi9UrWcN1n3ynyy9T9qr1CFmopQiwSQo+036a8F/3Y9KIFM +nhQWTBVcXkbmZtxpRRaD9rC6p+2aqfVNmjCDuYxjuGmTqL/0eayRefk0QwT5S51L +ea5WlmQtUbCpewnNiNrz +-----END CERTIFICATE----- diff --git a/config/certs/ca.key b/config/certs/ca.key new file mode 100644 index 00000000..ea1b62b9 --- /dev/null +++ b/config/certs/ca.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQD43Co0NROYEeeZ +WWH4O3m+V7U+1/4DVuAm+9u1bxqiOnliE24wm9+gk3HEwdr6pMGTfMWS8BMmqUpj +jgFVK4Tcur87Cqjq7XDe8j7h5Ipi8yVUAgqF4wesIZpGtxXRZwwGWwRu38zX5CAa +9n9Xbp0Y7tDdINRk0vLCp7VQbd2NVbMXgqygJAaAImdNfrddAmojWJ92LCT5HKcD +Nq8Z62VwtLqOUePiEJxm1sUts9tTsX8XlyLljz2aoWcX+Gzin8HkOftnpYeHptDL +26Q2FyW7TYZR4oFuhU6FQ/YPfCsEm/sxVC7BIBWal4DF29ZiivWvWk+wBNq03LxR +1/TvJtOdAgMBAAECggEAHfFKQq3VhbPhzIRXUqlqpImjt8P/6XbyfBOhroWHttfs +8u1TkhWvJSrtwrbFxOfol/j97LK7cH4TV+x7dzvykycFC0g6ZXSG4sxS30Btm3+V +5jMOARNSllYUsfLQH+STgHrttZ7H5CdWlZKYZR9crOX5y3a/whcqOQbkvKCmH522 +CUWeoabELLFGh6sGkQ6dXMFMS/fzT9MK3gxoLYYoTum5vJN2HYxMBerTbc3wXPtN +/BY8gStkv9BtbovPqsHUWeWK1fXphqysfKE79k9zS9PPz8ErEPhhT44blIo0BKCp +akw4DjHXbHhFq8Onh+3NemTLIhx++8z0FgL+6jYdIQKBgQD7BnNhHSxRm/smvyVm +L3fOX22fL1ZruY9sV4h3pK7KEqMLoUfrOpBN1AiQd/Xb/XNGMrHSPdZGL+tSS7vJ +2ibcSJWGuvuhA8HXpSPRUE9ao5cdRtEOTpy0FncMYaa7XYFoi1NOH/xl0uhd2u2i +uqL122tafZ9cP8fzvrOICa9ydwKBgQD9yrq3D3m5oIj596jDUsX2jxCKRzdnGLu3 +/qRnnfsN07524Ydqo8lpQMU+u2ljJhNFjcBqTOjSsRnnd1qQHqxZ18ABgOteAws5 +FlfLNQ7nupZu0IMd3b2WoCqoUOzLBsofUnMUPGRZLI2QCxfS1gzgp57IpY1egHe9 +1HL2Ht/7iwKBgCCeYffIlq20GxqZ0/5HRMYoWFJLEGvHHP/zD+ScHapcaZtlRbcn +UQEMAGDldak0cfo5NCohpupP58A13x0Hn+0X8XYkbfYqStH+v3y/dtBMWaKQRTIa +vPoZwTZ2qffG2r3+/MA2H9ILae5oOGDg57QS4wxLLp4KG61spl+TAp1ZAoGAbtG8 +UD0gsO6pgUUkWw3kxXHZDhhk187UAVbe4SP5wSLpsy1tnSIhy6TtvCPHs/SnS6LH +F2O38nBE2G6ki/Po3F46SC3MGMQJfYcFFQV5GbS/BWplJoBxzbjoh5C2pTy0u5Kc +D4UDaHDs6638XCL9goeO7RxlK5O7NZf0DwaSVVsCgYB6JQoVjob9kRIkko1ob63W +idsWNZtG3Py1gx61lBPk/Lz/bxJKlsnWgvInjGgFQ0o0g98hnd4H3O89BPwyAgKa +dIKfW/2k7SG0B+wB1xhnaBtHvJ3tWdqcH9wcTbOQ4LbT1OlvEQYc2B7cYicS22oZ +1lUUI2e9WKSTzwHk+nZQEw== +-----END PRIVATE KEY----- diff --git a/config/install_agent.tpl b/config/install_agent.tpl index 0df7fab0..47c365f8 100644 --- a/config/install_agent.tpl +++ b/config/install_agent.tpl @@ -115,8 +115,8 @@ agent_exc="${install_path}/agent/agent-${os}-${arch}" #agent可执行文件 agent_exsit="true" if [ ! -d "${install_path}/agent" ]; then - printf "\n* mkdir ${install_path}/agent" - $sudo_cmd mkdir "${install_path}/agent" + printf "\n* mkdir -p ${install_path}/agent" + $sudo_cmd mkdir -p "${install_path}/agent" agent_exsit="false" fi @@ -161,7 +161,7 @@ rm -f ${agent} ################## # save cert ################## -$sudo_cmd mkdir config +$sudo_cmd mkdir -p config $sudo_cmd sh -c "echo '${ca_crt}' > ./config/ca.crt" $sudo_cmd sh -c "echo '${client_crt}' > ./config/client.crt" $sudo_cmd sh -c "echo '${client_key}' > ./config/client.key" @@ -184,9 +184,9 @@ api: enabled: true tls: enabled: true - cert_file: "${install_path}/agent/config/client.crt" - key_file: "${install_path}/agent/config/client.key" - ca_file: "${install_path}/agent/config/ca.crt" + cert_file: "config/client.crt" + key_file: "config/client.key" + ca_file: "config/ca.crt" skip_insecure_verify: false network: binding: \$[[env.API_BINDING]] @@ -213,10 +213,10 @@ fi $sudo_cmd chmod +x $agent_exc #try to stop and uninstall service -if [[ "$agent_exsit" == "true" ]]; then +if [[ -e /etc/systemd/system/agent.service ]]; then printf "\n* stop && uninstall service\n" - $sudo_cmd $agent_exc -service stop - $sudo_cmd $agent_exc -service uninstall + $sudo_cmd $agent_exc -service stop &>/dev/null + $sudo_cmd $agent_exc -service uninstall &>/dev/null fi printf "\n* start install service\n" diff --git a/console.yml b/console.yml index 8c0cff79..7b80fc8b 100644 --- a/console.yml +++ b/console.yml @@ -1,10 +1,10 @@ path.configs: "config" configs.auto_reload: true -#env: -# INFINI_CONSOLE_ENDPOINT: "http://192.168.3.9:9000" -# INGEST_CLUSTER_ENDPOINT: "http://192.168.3.9:9210" -# INGEST_CLUSTER_CREDENTIAL_ID: chjkp9dath21f1ae9tq0 +env: + INFINI_CONSOLE_ENDPOINT: "http://127.0.0.1:9000" + INGEST_CLUSTER_ENDPOINT: "https://127.0.0.1:9200" + INGEST_CLUSTER_CREDENTIAL_ID: chjkp9dath21f1ae9tq0 web: enabled: true @@ -71,12 +71,12 @@ badger: # redirect_url: "" # scopes: [] -#agent: -# setup: -# download_url: "https://release.infinilabs.com/agent/snapshot" -# version: 0.5.0_NIGHTLY-157 -# ca_cert: "/opt/config/certs/ca.crt" -# ca_key: "/opt/config/certs/ca.key" -# console_endpoint: $[[env.INFINI_CONSOLE_ENDPOINT]] -# ingest_cluster_endpoint: $[[env.INGEST_CLUSTER_ENDPOINT]] -# ingest_cluster_credential_id: $[[env.INGEST_CLUSTER_CREDENTIAL_ID]] \ No newline at end of file +agent: + setup: + download_url: "https://release.infinilabs.com/agent/stable" + version: 0.5.0-214 + ca_cert: "config/certs/ca.crt" + ca_key: "config/certs/ca.key" + console_endpoint: $[[env.INFINI_CONSOLE_ENDPOINT]] + ingest_cluster_endpoint: $[[env.INGEST_CLUSTER_ENDPOINT]] + ingest_cluster_credential_id: $[[env.INGEST_CLUSTER_CREDENTIAL_ID]] \ No newline at end of file From a854e410a5443b7b3e90455bf8dc1a57a76089c2 Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 8 Jun 2023 14:24:55 +0800 Subject: [PATCH 3/7] fix task with certs --- Jenkinsfile-linux-amd64 | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile-linux-amd64 b/Jenkinsfile-linux-amd64 index 86249276..8bca75f8 100644 --- a/Jenkinsfile-linux-amd64 +++ b/Jenkinsfile-linux-amd64 @@ -26,6 +26,7 @@ pipeline { sh 'cd /home/jenkins/go/src/infini.sh/console && git pull origin master && make config build-linux-amd64' sh label: 'copy-license', script: 'cd /home/jenkins/go/src/infini.sh/console && cp ../framework/LICENSE bin && cat ../framework/NOTICE NOTICE > bin/NOTICE' sh label: 'copy-configs', script: 'cd /home/jenkins/go/src/infini.sh/console && mkdir -p bin/config && cp config/*.json bin/config && cp config/*.tpl bin/config' + sh label: 'copy-certs', script: 'cd /home/jenkins/go/src/infini.sh/console && cp -rf config/certs bin/config' sh label: 'package-linux-amd64', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-amd64.tar.gz console-linux-amd64 console.yml LICENSE NOTICE config' archiveArtifacts artifacts: 'console-$VERSION-$BUILD_NUMBER-*.*', fingerprint: true, followSymlinks: true, onlyIfSuccessful: false From 25caee910934b69c056d70d090d34aa152af5f49 Mon Sep 17 00:00:00 2001 From: liugq Date: Thu, 8 Jun 2023 14:33:44 +0800 Subject: [PATCH 4/7] do not panic when agent was not configured --- modules/agent/common/config.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/agent/common/config.go b/modules/agent/common/config.go index ac7e255e..33319101 100644 --- a/modules/agent/common/config.go +++ b/modules/agent/common/config.go @@ -7,6 +7,7 @@ package common import ( "infini.sh/console/modules/agent/model" "infini.sh/framework/core/env" + log "src/github.com/cihub/seelog" ) @@ -14,7 +15,7 @@ func GetAgentConfig() *model.AgentConfig { agentCfg := &model.AgentConfig{} _, err := env.ParseConfig("agent", agentCfg ) if err != nil { - panic(err) + log.Error("agent config not found: %v", err) } return agentCfg } \ No newline at end of file From ec44fb0c666955c4dad726475a8e0edbef60b311 Mon Sep 17 00:00:00 2001 From: liugq Date: Thu, 8 Jun 2023 14:49:53 +0800 Subject: [PATCH 5/7] add dashboard init script --- config/initialization.tpl | 1744 ++++++++++++++++++++++++++++++++++ config/initialization_v5.tpl | 1744 ++++++++++++++++++++++++++++++++++ config/initialization_v6.tpl | 1743 +++++++++++++++++++++++++++++++++ 3 files changed, 5231 insertions(+) diff --git a/config/initialization.tpl b/config/initialization.tpl index 9a798041..671a5aa5 100644 --- a/config/initialization.tpl +++ b/config/initialization.tpl @@ -2192,6 +2192,1750 @@ POST $[[INDEX_PREFIX]]view/_doc/cb34sfl6psfiqtovhpt4 } +POST $[[INDEX_PREFIX]]layout/_doc/cgjoqud3q95rinbbe1l0 +{ + "id": "cgjoqud3q95rinbbe1l0", + "created": "2023-04-01T01:48:41.54255458Z", + "updated": "2023-05-11T03:15:45.877680081Z", + "name": "Platform Overview", + "description": "", + "creator": { + "name": "admin", + "id": "cft0jdtath25bt7npsgg" + }, + "view_id": "", + "config": { + "cols": 12, + "globalQueries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": ".infini_metrics*", + "time_field": "timestamp", + "time_range": { + "from": "now-30m", + "to": "now" + } + }, + "global_queries": {}, + "row_height": 60, + "widgets": [ + { + "formatter": "number", + "position": { + "h": 4, + "w": 3, + "x": 9, + "y": 5 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "payload.elasticsearch.cluster_stats.nodes.jvm.versions.version", + "limit": 5 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics" + ], + "time_field": "timestamp" + }, + "type": "pie" + } + ], + "title": "JDK Version Distribution" + }, + { + "formatter": "number", + "position": { + "h": 4, + "w": 3, + "x": 6, + "y": 5 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "payload.elasticsearch.cluster_health.status", + "limit": 5 + } + ], + "items": [ + { + "field": "metadata.labels.cluster_id", + "name": "a", + "statistic": "cardinality" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics" + ], + "time_field": "timestamp" + }, + "type": "pie" + } + ], + "title": "Health Status" + }, + { + "formatter": "number", + "position": { + "h": 4, + "w": 3, + "x": 0, + "y": 5 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "location.provider", + "limit": 5 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_cluster" + ] + }, + "type": "pie" + } + ], + "title": "Service Provider" + }, + { + "formatter": "number", + "position": { + "h": 4, + "w": 3, + "x": 3, + "y": 5 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "distribution", + "limit": 5 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_cluster" + ] + }, + "type": "pie" + } + ], + "title": "Search Engine Distribution" + }, + { + "data_type": "normal", + "formatter": "number", + "order": "desc", + "position": { + "h": 5, + "w": 6, + "x": 6, + "y": 14 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "metadata.labels.node_id", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.elasticsearch.node_stats.os.cpu.load_average.15m", + "name": "a", + "statistic": "max" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.name\": {\n \"value\": \"node_stats\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "column" + } + ], + "size": 10, + "title": "Node CPU Load TOP10" + }, + { + "data_type": "normal", + "formatter": "percent", + "order": "desc", + "position": { + "h": 5, + "w": 6, + "x": 0, + "y": 9 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "(b-a)/b*100", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 50 + }, + { + "field": "metadata.labels.node_name", + "limit": 500 + } + ], + "items": [ + { + "field": "payload.elasticsearch.node_stats.fs.total.available_in_bytes", + "name": "a", + "statistic": "max" + }, + { + "field": "payload.elasticsearch.node_stats.fs.total.total_in_bytes", + "name": "b", + "statistic": "max" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "column" + } + ], + "size": 10, + "title": "Node Disk Usage TOP10" + }, + { + "data_type": "timeseries", + "formatter": "percent", + "position": { + "h": 5, + "w": 6, + "x": 6, + "y": 9 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 50 + }, + { + "field": "metadata.labels.node_name", + "limit": 500 + } + ], + "items": [ + { + "field": "metadata.datatype", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "column" + } + ], + "title": "Node JVM Heap Usage TOP10" + }, + { + "data_type": "normal", + "formatter": "bytes", + "order": "desc", + "position": { + "h": 5, + "w": 6, + "x": 0, + "y": 14 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 50 + }, + { + "field": "metadata.labels.index_name", + "limit": 1000 + } + ], + "items": [ + { + "field": "payload.elasticsearch.index_stats.shard_info.store_in_bytes", + "name": "a", + "statistic": "max" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "column" + } + ], + "size": 10, + "title": "Index Shard Size TOP10" + } + ] + }, + "type": "workspace", + "is_fixed": true +} + +POST $[[INDEX_PREFIX]]layout/_doc/cgjpvt53q95r17vbdteg +{ + "id": "cgjpvt53q95r17vbdteg", + "created": "2023-04-01T03:07:32.165611988Z", + "updated": "2023-05-25T07:57:44.860138405Z", + "name": "Metrics&Logging Overview", + "description": "", + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + }, + "view_id": "", + "config": { + "cols": 12, + "globalQueries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": ".infini_metrics*", + "time_field": "timestamp" + }, + "global_queries": {}, + "row_height": 60, + "widgets": [ + { + "drilling": { + "new_tab_switch": false, + "url": "" + }, + "formatter": "number", + "position": { + "h": 4, + "w": 12, + "x": 0, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "payload.elasticsearch.cluster_stats.indices.docs.deleted", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "Log Collection" + }, + { + "formatter": "number", + "position": { + "h": 4, + "w": 6, + "x": 6, + "y": 8 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "agent.major_ip", + "limit": 5 + } + ], + "items": [ + { + "field": "payload.host.network_sockets.all.established", + "name": "a", + "statistic": "p99" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "TCP Connection" + }, + { + "formatter": "number", + "position": { + "h": 4, + "w": 6, + "x": 0, + "y": 12 + }, + "series": [ + { + "metric": { + "bucket_size": "30s", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "agent.major_ip", + "limit": 5 + } + ], + "items": [ + { + "field": "payload.host.network_sockets.all.connections", + "name": "a", + "statistic": "min" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "area" + } + ], + "title": "TCP Connections" + }, + { + "data_type": "timeseries", + "formatter": "number", + "position": { + "h": 4, + "w": 12, + "x": 0, + "y": 4 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "metadata.category", + "limit": 50 + }, + { + "field": "metadata.name", + "limit": 50 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "column" + } + ], + "title": "Log Data Distribution" + }, + { + "formatter": "bytes", + "position": { + "h": 4, + "w": 6, + "x": 0, + "y": 8 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "agent.major_ip", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.host.network_summary.in.bytes", + "name": "a", + "statistic": "derivative" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"host\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"network_summary\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "area" + } + ], + "title": "Network Traffic (In)" + }, + { + "formatter": "bytes", + "position": { + "h": 4, + "w": 6, + "x": 6, + "y": 12 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "agent.major_ip", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.host.network_summary.out.bytes", + "name": "a", + "statistic": "derivative" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"host\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"network_summary\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "Network Traffic (Out)" + } + ] + }, + "type": "workspace", + "is_fixed": true + } + +POST $[[INDEX_PREFIX]]layout/_doc/cgjqcg53q95r17vbfo10 +{ + "id": "cgjqcg53q95r17vbfo10", + "created": "2023-04-01T03:34:24.919282378Z", + "updated": "2023-06-07T07:54:46.991959699Z", + "name": "INFINI Gateway", + "description": "", + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + }, + "view_id": "", + "config": { + "cols": 12, + "globalQueries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": ".infini_metrics*", + "time_field": "timestamp", + "time_range": { + "from": "now-30m", + "to": "now" + } + }, + "global_queries": {}, + "row_height": 60, + "widgets": [ + { + "formatter": "number", + "position": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "10s", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "metadata.labels.name", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.instance.entry.my_es_entry.open_connections", + "name": "a", + "statistic": "max" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"instance\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"gateway\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "Gateway Client Connections" + }, + { + "bucket_size": "auto", + "drilling": {}, + "formatter": "percent", + "position": { + "h": 4, + "w": 4, + "x": 4, + "y": 0 + }, + "series": [ + { + "metric": { + "formula": "a", + "groups": [ + { + "field": "metadata.labels.name", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.instance.system.cpu", + "name": "a", + "statistic": "max" + } + ], + "name": "Overall CPU" + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"instance\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"gateway\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "line" + }, + { + "metric": { + "formula": "(a + b) / c * 100", + "groups": [ + { + "field": "metadata.labels.name", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.instance.system.sys_in_ms", + "name": "a", + "statistic": "derivative" + }, + { + "field": "payload.instance.system.user_in_ms", + "name": "b", + "statistic": "derivative" + }, + { + "field": "timestamp", + "name": "c", + "statistic": "derivative" + } + ], + "name": "Real-Time CPU" + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"instance\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"gateway\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "Gateway CPU Utilization" + }, + { + "formatter": "bytes", + "position": { + "h": 4, + "w": 4, + "x": 8, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "10s", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "metadata.labels.name", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.instance.system.mem", + "name": "a", + "statistic": "max" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"instance\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"gateway\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "Gateway Memory Usage" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 5, + "w": 12, + "x": 0, + "y": 4 + }, + "series": [ + { + "metric": { + "bucket_size": "10s", + "formula": "a", + "groups": [ + { + "field": "metadata.labels.name", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.instance.system.goroutines", + "name": "a", + "statistic": "max" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"instance\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"gateway\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "Gateway Goroutine" + }, + { + "formatter": "number", + "position": { + "h": 5, + "w": 12, + "x": 0, + "y": 9 + }, + "series": [ + { + "metric": { + "bucket_size": "10s", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "metadata.labels.name", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.instance.system.objects", + "name": "a", + "statistic": "max" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"instance\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"gateway\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "Gateway Active Objects" + } + ] + }, + "type": "workspace", + "is_fixed": true + } +POST $[[INDEX_PREFIX]]layout/_doc/ch0u5d53q95poj13629g +{ + "id": "ch0u5d53q95poj13629g", + "created": "2023-04-21T01:11:16.129965374Z", + "updated": "2023-05-25T12:23:10.993444364Z", + "name": "Clusters Overview", + "description": "", + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + }, + "view_id": "", + "config": { + "cols": 12, + "global_queries": {}, + "row_height": 60, + "widgets": [ + { + "formatter": "number", + "position": { + "h": 3, + "w": 3, + "x": 3, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "items": [ + { + "field": "agent.major_ip", + "name": "a", + "statistic": "cardinality" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "number" + } + ], + "title": "Number Of Hosts" + }, + { + "drilling": {}, + "page_size": 10, + "position": { + "h": 5, + "w": 12, + "x": 0, + "y": 4 + }, + "series": [ + { + "columns": [ + { + "display": "ClusterName", + "name": "name", + "type": "string" + }, + { + "name": "distribution", + "type": "string" + }, + { + "name": "endpoint", + "type": "string" + }, + { + "display": "Time", + "formatter": "time", + "name": "created", + "type": "date" + } + ], + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_cluster" + ] + }, + "type": "table" + } + ], + "title": "Cluster List" + }, + { + "formatter": "number", + "position": { + "h": 3, + "w": 3, + "x": 6, + "y": 1 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "items": [ + { + "field": "metadata.labels.node_id", + "name": "a", + "statistic": "cardinality" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "number" + } + ], + "title": "Number Of Nodes" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 3, + "w": 3, + "x": 0, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "metadata.labels.cluster_id", + "name": "a", + "statistic": "cardinality" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "number" + } + ], + "title": "Number of Clusters" + } + ] + }, + "type": "workspace", + "is_fixed": true +} +POST $[[INDEX_PREFIX]]layout/_doc/cgjqcg53q95r17vbfo10 +{ + "id": "chnmaht3q95ph02nfpsg", + "created": "2023-05-25T13:40:23.821091313Z", + "updated": "2023-06-08T03:49:09.247667399Z", + "name": "Requests-analysis", + "description": "", + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + }, + "view_id": "", + "config": { + "cols": 12, + "global_queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_requests_logging*" + ], + "time_field": "timestamp" + }, + "row_height": 60, + "widgets": [ + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 2, + "w": 2, + "x": 0, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "number" + } + ], + "title": "Total Requests" + }, + { + "bucket_size": "auto", + "drilling": {}, + "formatter": "number", + "position": { + "h": 4, + "w": 12, + "x": 0, + "y": 5 + }, + "series": [ + { + "metric": { + "formula": "a", + "items": [ + { + "field": "response.elapsed", + "name": "a", + "statistic": "p99" + } + ], + "name": "P99" + }, + "queries": { + "indices": [] + }, + "type": "line" + }, + { + "metric": { + "formula": "a", + "items": [ + { + "field": "response.elapsed", + "name": "a", + "statistic": "avg" + } + ], + "name": "AVG" + }, + "queries": { + "indices": [] + }, + "type": "line" + }, + { + "metric": { + "formula": "a", + "items": [ + { + "field": "response.elapsed", + "name": "a", + "statistic": "p50" + } + ], + "name": "P50" + }, + "queries": { + "indices": [] + }, + "type": "line" + } + ], + "title": "Request Latency" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 3, + "w": 12, + "x": 0, + "y": 2 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "area" + } + ], + "title": "Request rate" + }, + { + "data_type": "timeseries", + "drilling": {}, + "formatter": "number", + "position": { + "h": 5, + "w": 12, + "x": 0, + "y": 14 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "request.path", + "limit": 5 + } + ], + "items": [ + { + "field": "response.elapsed", + "name": "a", + "statistic": "p95" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "column" + } + ], + "title": "wrong" + }, + { + "bucket_size": "auto", + "data_type": "timeseries", + "drilling": {}, + "formatter": "number", + "position": { + "h": 5, + "w": 12, + "x": 0, + "y": 9 + }, + "series": [ + { + "metric": { + "formula": "a", + "groups": [ + { + "field": "request.path", + "limit": 6 + } + ], + "items": [ + { + "field": "request.path", + "name": "a", + "statistic": "count" + } + ], + "name": "" + }, + "queries": { + "indices": [] + }, + "type": "column" + } + ], + "title": "Request rate group by path" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 2, + "w": 2, + "x": 2, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_requests_logging*" + ], + "query": "{\n \"bool\": {\n \"filter\": [\n {\"term\": {\"response.cached\": {\"value\": \"true\" } }}\n\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "number" + } + ], + "title": "Cached Requests" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 4, + "w": 2, + "x": 0, + "y": 23 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "response.status_code", + "limit": 5 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "pie" + } + ], + "title": "Response code" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 2, + "w": 2, + "x": 10, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_requests_logging*" + ], + "query": "{\n \"range\": {\n \"response.elapsed\": {\n \"gte\": 1000\n }\n }\n }", + "time_field": "timestamp" + }, + "type": "number" + } + ], + "title": "Slow Queries" + }, + { + "drilling": {}, + "page_size": 10, + "position": { + "h": 5, + "w": 12, + "x": 0, + "y": 27 + }, + "series": [ + { + "columns": [ + { + "name": "flow.from", + "type": "string" + }, + { + "name": "flow.relay", + "type": "string" + }, + { + "name": "flow.to", + "type": "string" + }, + { + "name": "request.method", + "type": "string" + }, + { + "name": "request.path", + "type": "string" + }, + { + "name": "response.status_code", + "type": "number" + } + ], + "queries": { + "indices": [] + }, + "type": "table" + } + ], + "title": "Request detail" + }, + { + "data_type": "timeseries", + "drilling": {}, + "formatter": "bytes", + "position": { + "h": 4, + "w": 5, + "x": 7, + "y": 19 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "remote_ip", + "limit": 5 + } + ], + "items": [ + { + "field": "response.body_length", + "name": "a", + "statistic": "sum" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "column" + } + ], + "title": "Response size" + }, + { + "data_type": "timeseries", + "drilling": {}, + "formatter": "bytes", + "position": { + "h": 4, + "w": 5, + "x": 2, + "y": 19 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "remote_ip", + "limit": 5 + } + ], + "items": [ + { + "field": "request.body_length", + "name": "a", + "statistic": "sum" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "column" + } + ], + "title": "Request size" + }, + { + "data_type": "timeseries", + "drilling": {}, + "formatter": "number", + "position": { + "h": 4, + "w": 5, + "x": 2, + "y": 23 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "flow.to", + "limit": 5 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "column" + } + ], + "title": "Request upstreams" + }, + { + "data_type": "timeseries", + "drilling": {}, + "formatter": "number", + "position": { + "h": 4, + "w": 5, + "x": 7, + "y": 23 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "request.method", + "limit": 5 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "column" + } + ], + "title": "Request methods" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 4, + "w": 2, + "x": 0, + "y": 19 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "flow.from", + "limit": 5 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "pie" + } + ], + "title": "Clients ip" + }, + { + "drilling": {}, + "formatter": "bytes", + "position": { + "h": 2, + "w": 2, + "x": 8, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "response.body_length", + "name": "a", + "statistic": "sum" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "number" + } + ], + "title": "Response traffic" + }, + { + "drilling": {}, + "formatter": "bytes", + "position": { + "h": 2, + "w": 2, + "x": 6, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "request.body_length", + "name": "a", + "statistic": "sum" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "number" + } + ], + "title": "Request traffic" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 2, + "w": 2, + "x": 4, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "remote_ip", + "name": "a", + "statistic": "cardinality" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "number" + } + ], + "title": "Clients" + } + ] + }, + "type": "workspace", + "is_fixed": true +} GET / diff --git a/config/initialization_v5.tpl b/config/initialization_v5.tpl index 78e16a40..a86e8452 100644 --- a/config/initialization_v5.tpl +++ b/config/initialization_v5.tpl @@ -1589,6 +1589,1750 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/cb34sfl6psfiqtovhpt4 } } +POST $[[INDEX_PREFIX]]layout/doc/cgjoqud3q95rinbbe1l0 +{ + "id": "cgjoqud3q95rinbbe1l0", + "created": "2023-04-01T01:48:41.54255458Z", + "updated": "2023-05-11T03:15:45.877680081Z", + "name": "Platform Overview", + "description": "", + "creator": { + "name": "admin", + "id": "cft0jdtath25bt7npsgg" + }, + "view_id": "", + "config": { + "cols": 12, + "globalQueries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": ".infini_metrics*", + "time_field": "timestamp", + "time_range": { + "from": "now-30m", + "to": "now" + } + }, + "global_queries": {}, + "row_height": 60, + "widgets": [ + { + "formatter": "number", + "position": { + "h": 4, + "w": 3, + "x": 9, + "y": 5 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "payload.elasticsearch.cluster_stats.nodes.jvm.versions.version", + "limit": 5 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics" + ], + "time_field": "timestamp" + }, + "type": "pie" + } + ], + "title": "JDK Version Distribution" + }, + { + "formatter": "number", + "position": { + "h": 4, + "w": 3, + "x": 6, + "y": 5 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "payload.elasticsearch.cluster_health.status", + "limit": 5 + } + ], + "items": [ + { + "field": "metadata.labels.cluster_id", + "name": "a", + "statistic": "cardinality" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics" + ], + "time_field": "timestamp" + }, + "type": "pie" + } + ], + "title": "Health Status" + }, + { + "formatter": "number", + "position": { + "h": 4, + "w": 3, + "x": 0, + "y": 5 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "location.provider", + "limit": 5 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_cluster" + ] + }, + "type": "pie" + } + ], + "title": "Service Provider" + }, + { + "formatter": "number", + "position": { + "h": 4, + "w": 3, + "x": 3, + "y": 5 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "distribution", + "limit": 5 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_cluster" + ] + }, + "type": "pie" + } + ], + "title": "Search Engine Distribution" + }, + { + "data_type": "normal", + "formatter": "number", + "order": "desc", + "position": { + "h": 5, + "w": 6, + "x": 6, + "y": 14 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "metadata.labels.node_id", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.elasticsearch.node_stats.os.cpu.load_average.15m", + "name": "a", + "statistic": "max" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.name\": {\n \"value\": \"node_stats\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "column" + } + ], + "size": 10, + "title": "Node CPU Load TOP10" + }, + { + "data_type": "normal", + "formatter": "percent", + "order": "desc", + "position": { + "h": 5, + "w": 6, + "x": 0, + "y": 9 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "(b-a)/b*100", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 50 + }, + { + "field": "metadata.labels.node_name", + "limit": 500 + } + ], + "items": [ + { + "field": "payload.elasticsearch.node_stats.fs.total.available_in_bytes", + "name": "a", + "statistic": "max" + }, + { + "field": "payload.elasticsearch.node_stats.fs.total.total_in_bytes", + "name": "b", + "statistic": "max" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "column" + } + ], + "size": 10, + "title": "Node Disk Usage TOP10" + }, + { + "data_type": "timeseries", + "formatter": "percent", + "position": { + "h": 5, + "w": 6, + "x": 6, + "y": 9 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 50 + }, + { + "field": "metadata.labels.node_name", + "limit": 500 + } + ], + "items": [ + { + "field": "metadata.datatype", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "column" + } + ], + "title": "Node JVM Heap Usage TOP10" + }, + { + "data_type": "normal", + "formatter": "bytes", + "order": "desc", + "position": { + "h": 5, + "w": 6, + "x": 0, + "y": 14 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 50 + }, + { + "field": "metadata.labels.index_name", + "limit": 1000 + } + ], + "items": [ + { + "field": "payload.elasticsearch.index_stats.shard_info.store_in_bytes", + "name": "a", + "statistic": "max" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "column" + } + ], + "size": 10, + "title": "Index Shard Size TOP10" + } + ] + }, + "type": "workspace", + "is_fixed": true +} + +POST $[[INDEX_PREFIX]]layout/doc/cgjpvt53q95r17vbdteg +{ + "id": "cgjpvt53q95r17vbdteg", + "created": "2023-04-01T03:07:32.165611988Z", + "updated": "2023-05-25T07:57:44.860138405Z", + "name": "Metrics&Logging Overview", + "description": "", + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + }, + "view_id": "", + "config": { + "cols": 12, + "globalQueries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": ".infini_metrics*", + "time_field": "timestamp" + }, + "global_queries": {}, + "row_height": 60, + "widgets": [ + { + "drilling": { + "new_tab_switch": false, + "url": "" + }, + "formatter": "number", + "position": { + "h": 4, + "w": 12, + "x": 0, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "payload.elasticsearch.cluster_stats.indices.docs.deleted", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "Log Collection" + }, + { + "formatter": "number", + "position": { + "h": 4, + "w": 6, + "x": 6, + "y": 8 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "agent.major_ip", + "limit": 5 + } + ], + "items": [ + { + "field": "payload.host.network_sockets.all.established", + "name": "a", + "statistic": "p99" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "TCP Connection" + }, + { + "formatter": "number", + "position": { + "h": 4, + "w": 6, + "x": 0, + "y": 12 + }, + "series": [ + { + "metric": { + "bucket_size": "30s", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "agent.major_ip", + "limit": 5 + } + ], + "items": [ + { + "field": "payload.host.network_sockets.all.connections", + "name": "a", + "statistic": "min" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "area" + } + ], + "title": "TCP Connections" + }, + { + "data_type": "timeseries", + "formatter": "number", + "position": { + "h": 4, + "w": 12, + "x": 0, + "y": 4 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "metadata.category", + "limit": 50 + }, + { + "field": "metadata.name", + "limit": 50 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "column" + } + ], + "title": "Log Data Distribution" + }, + { + "formatter": "bytes", + "position": { + "h": 4, + "w": 6, + "x": 0, + "y": 8 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "agent.major_ip", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.host.network_summary.in.bytes", + "name": "a", + "statistic": "derivative" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"host\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"network_summary\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "area" + } + ], + "title": "Network Traffic (In)" + }, + { + "formatter": "bytes", + "position": { + "h": 4, + "w": 6, + "x": 6, + "y": 12 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "agent.major_ip", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.host.network_summary.out.bytes", + "name": "a", + "statistic": "derivative" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"host\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"network_summary\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "Network Traffic (Out)" + } + ] + }, + "type": "workspace", + "is_fixed": true + } + +POST $[[INDEX_PREFIX]]layout/doc/cgjqcg53q95r17vbfo10 +{ + "id": "cgjqcg53q95r17vbfo10", + "created": "2023-04-01T03:34:24.919282378Z", + "updated": "2023-06-07T07:54:46.991959699Z", + "name": "INFINI Gateway", + "description": "", + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + }, + "view_id": "", + "config": { + "cols": 12, + "globalQueries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": ".infini_metrics*", + "time_field": "timestamp", + "time_range": { + "from": "now-30m", + "to": "now" + } + }, + "global_queries": {}, + "row_height": 60, + "widgets": [ + { + "formatter": "number", + "position": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "10s", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "metadata.labels.name", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.instance.entry.my_es_entry.open_connections", + "name": "a", + "statistic": "max" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"instance\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"gateway\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "Gateway Client Connections" + }, + { + "bucket_size": "auto", + "drilling": {}, + "formatter": "percent", + "position": { + "h": 4, + "w": 4, + "x": 4, + "y": 0 + }, + "series": [ + { + "metric": { + "formula": "a", + "groups": [ + { + "field": "metadata.labels.name", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.instance.system.cpu", + "name": "a", + "statistic": "max" + } + ], + "name": "Overall CPU" + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"instance\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"gateway\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "line" + }, + { + "metric": { + "formula": "(a + b) / c * 100", + "groups": [ + { + "field": "metadata.labels.name", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.instance.system.sys_in_ms", + "name": "a", + "statistic": "derivative" + }, + { + "field": "payload.instance.system.user_in_ms", + "name": "b", + "statistic": "derivative" + }, + { + "field": "timestamp", + "name": "c", + "statistic": "derivative" + } + ], + "name": "Real-Time CPU" + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"instance\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"gateway\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "Gateway CPU Utilization" + }, + { + "formatter": "bytes", + "position": { + "h": 4, + "w": 4, + "x": 8, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "10s", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "metadata.labels.name", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.instance.system.mem", + "name": "a", + "statistic": "max" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"instance\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"gateway\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "Gateway Memory Usage" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 5, + "w": 12, + "x": 0, + "y": 4 + }, + "series": [ + { + "metric": { + "bucket_size": "10s", + "formula": "a", + "groups": [ + { + "field": "metadata.labels.name", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.instance.system.goroutines", + "name": "a", + "statistic": "max" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"instance\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"gateway\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "Gateway Goroutine" + }, + { + "formatter": "number", + "position": { + "h": 5, + "w": 12, + "x": 0, + "y": 9 + }, + "series": [ + { + "metric": { + "bucket_size": "10s", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "metadata.labels.name", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.instance.system.objects", + "name": "a", + "statistic": "max" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"instance\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"gateway\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "Gateway Active Objects" + } + ] + }, + "type": "workspace", + "is_fixed": true + } +POST $[[INDEX_PREFIX]]layout/doc/ch0u5d53q95poj13629g +{ + "id": "ch0u5d53q95poj13629g", + "created": "2023-04-21T01:11:16.129965374Z", + "updated": "2023-05-25T12:23:10.993444364Z", + "name": "Clusters Overview", + "description": "", + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + }, + "view_id": "", + "config": { + "cols": 12, + "global_queries": {}, + "row_height": 60, + "widgets": [ + { + "formatter": "number", + "position": { + "h": 3, + "w": 3, + "x": 3, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "items": [ + { + "field": "agent.major_ip", + "name": "a", + "statistic": "cardinality" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "number" + } + ], + "title": "Number Of Hosts" + }, + { + "drilling": {}, + "page_size": 10, + "position": { + "h": 5, + "w": 12, + "x": 0, + "y": 4 + }, + "series": [ + { + "columns": [ + { + "display": "ClusterName", + "name": "name", + "type": "string" + }, + { + "name": "distribution", + "type": "string" + }, + { + "name": "endpoint", + "type": "string" + }, + { + "display": "Time", + "formatter": "time", + "name": "created", + "type": "date" + } + ], + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_cluster" + ] + }, + "type": "table" + } + ], + "title": "Cluster List" + }, + { + "formatter": "number", + "position": { + "h": 3, + "w": 3, + "x": 6, + "y": 1 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "items": [ + { + "field": "metadata.labels.node_id", + "name": "a", + "statistic": "cardinality" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "number" + } + ], + "title": "Number Of Nodes" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 3, + "w": 3, + "x": 0, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "metadata.labels.cluster_id", + "name": "a", + "statistic": "cardinality" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "number" + } + ], + "title": "Number of Clusters" + } + ] + }, + "type": "workspace", + "is_fixed": true +} +POST $[[INDEX_PREFIX]]layout/doc/cgjqcg53q95r17vbfo10 +{ + "id": "chnmaht3q95ph02nfpsg", + "created": "2023-05-25T13:40:23.821091313Z", + "updated": "2023-06-08T03:49:09.247667399Z", + "name": "Requests-analysis", + "description": "", + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + }, + "view_id": "", + "config": { + "cols": 12, + "global_queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_requests_logging*" + ], + "time_field": "timestamp" + }, + "row_height": 60, + "widgets": [ + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 2, + "w": 2, + "x": 0, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "number" + } + ], + "title": "Total Requests" + }, + { + "bucket_size": "auto", + "drilling": {}, + "formatter": "number", + "position": { + "h": 4, + "w": 12, + "x": 0, + "y": 5 + }, + "series": [ + { + "metric": { + "formula": "a", + "items": [ + { + "field": "response.elapsed", + "name": "a", + "statistic": "p99" + } + ], + "name": "P99" + }, + "queries": { + "indices": [] + }, + "type": "line" + }, + { + "metric": { + "formula": "a", + "items": [ + { + "field": "response.elapsed", + "name": "a", + "statistic": "avg" + } + ], + "name": "AVG" + }, + "queries": { + "indices": [] + }, + "type": "line" + }, + { + "metric": { + "formula": "a", + "items": [ + { + "field": "response.elapsed", + "name": "a", + "statistic": "p50" + } + ], + "name": "P50" + }, + "queries": { + "indices": [] + }, + "type": "line" + } + ], + "title": "Request Latency" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 3, + "w": 12, + "x": 0, + "y": 2 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "area" + } + ], + "title": "Request rate" + }, + { + "data_type": "timeseries", + "drilling": {}, + "formatter": "number", + "position": { + "h": 5, + "w": 12, + "x": 0, + "y": 14 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "request.path", + "limit": 5 + } + ], + "items": [ + { + "field": "response.elapsed", + "name": "a", + "statistic": "p95" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "column" + } + ], + "title": "wrong" + }, + { + "bucket_size": "auto", + "data_type": "timeseries", + "drilling": {}, + "formatter": "number", + "position": { + "h": 5, + "w": 12, + "x": 0, + "y": 9 + }, + "series": [ + { + "metric": { + "formula": "a", + "groups": [ + { + "field": "request.path", + "limit": 6 + } + ], + "items": [ + { + "field": "request.path", + "name": "a", + "statistic": "count" + } + ], + "name": "" + }, + "queries": { + "indices": [] + }, + "type": "column" + } + ], + "title": "Request rate group by path" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 2, + "w": 2, + "x": 2, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_requests_logging*" + ], + "query": "{\n \"bool\": {\n \"filter\": [\n {\"term\": {\"response.cached\": {\"value\": \"true\" } }}\n\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "number" + } + ], + "title": "Cached Requests" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 4, + "w": 2, + "x": 0, + "y": 23 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "response.status_code", + "limit": 5 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "pie" + } + ], + "title": "Response code" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 2, + "w": 2, + "x": 10, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_requests_logging*" + ], + "query": "{\n \"range\": {\n \"response.elapsed\": {\n \"gte\": 1000\n }\n }\n }", + "time_field": "timestamp" + }, + "type": "number" + } + ], + "title": "Slow Queries" + }, + { + "drilling": {}, + "page_size": 10, + "position": { + "h": 5, + "w": 12, + "x": 0, + "y": 27 + }, + "series": [ + { + "columns": [ + { + "name": "flow.from", + "type": "string" + }, + { + "name": "flow.relay", + "type": "string" + }, + { + "name": "flow.to", + "type": "string" + }, + { + "name": "request.method", + "type": "string" + }, + { + "name": "request.path", + "type": "string" + }, + { + "name": "response.status_code", + "type": "number" + } + ], + "queries": { + "indices": [] + }, + "type": "table" + } + ], + "title": "Request detail" + }, + { + "data_type": "timeseries", + "drilling": {}, + "formatter": "bytes", + "position": { + "h": 4, + "w": 5, + "x": 7, + "y": 19 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "remote_ip", + "limit": 5 + } + ], + "items": [ + { + "field": "response.body_length", + "name": "a", + "statistic": "sum" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "column" + } + ], + "title": "Response size" + }, + { + "data_type": "timeseries", + "drilling": {}, + "formatter": "bytes", + "position": { + "h": 4, + "w": 5, + "x": 2, + "y": 19 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "remote_ip", + "limit": 5 + } + ], + "items": [ + { + "field": "request.body_length", + "name": "a", + "statistic": "sum" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "column" + } + ], + "title": "Request size" + }, + { + "data_type": "timeseries", + "drilling": {}, + "formatter": "number", + "position": { + "h": 4, + "w": 5, + "x": 2, + "y": 23 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "flow.to", + "limit": 5 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "column" + } + ], + "title": "Request upstreams" + }, + { + "data_type": "timeseries", + "drilling": {}, + "formatter": "number", + "position": { + "h": 4, + "w": 5, + "x": 7, + "y": 23 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "request.method", + "limit": 5 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "column" + } + ], + "title": "Request methods" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 4, + "w": 2, + "x": 0, + "y": 19 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "flow.from", + "limit": 5 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "pie" + } + ], + "title": "Clients ip" + }, + { + "drilling": {}, + "formatter": "bytes", + "position": { + "h": 2, + "w": 2, + "x": 8, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "response.body_length", + "name": "a", + "statistic": "sum" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "number" + } + ], + "title": "Response traffic" + }, + { + "drilling": {}, + "formatter": "bytes", + "position": { + "h": 2, + "w": 2, + "x": 6, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "request.body_length", + "name": "a", + "statistic": "sum" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "number" + } + ], + "title": "Request traffic" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 2, + "w": 2, + "x": 4, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "remote_ip", + "name": "a", + "statistic": "cardinality" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "number" + } + ], + "title": "Clients" + } + ] + }, + "type": "workspace", + "is_fixed": true +} GET / diff --git a/config/initialization_v6.tpl b/config/initialization_v6.tpl index 0ba0a059..726ec408 100644 --- a/config/initialization_v6.tpl +++ b/config/initialization_v6.tpl @@ -1664,7 +1664,1750 @@ POST $[[INDEX_PREFIX]]alert-rule/doc/cb34sfl6psfiqtovhpt4 "interval": "1m" } } +POST $[[INDEX_PREFIX]]layout/doc/cgjoqud3q95rinbbe1l0 +{ + "id": "cgjoqud3q95rinbbe1l0", + "created": "2023-04-01T01:48:41.54255458Z", + "updated": "2023-05-11T03:15:45.877680081Z", + "name": "Platform Overview", + "description": "", + "creator": { + "name": "admin", + "id": "cft0jdtath25bt7npsgg" + }, + "view_id": "", + "config": { + "cols": 12, + "globalQueries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": ".infini_metrics*", + "time_field": "timestamp", + "time_range": { + "from": "now-30m", + "to": "now" + } + }, + "global_queries": {}, + "row_height": 60, + "widgets": [ + { + "formatter": "number", + "position": { + "h": 4, + "w": 3, + "x": 9, + "y": 5 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "payload.elasticsearch.cluster_stats.nodes.jvm.versions.version", + "limit": 5 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics" + ], + "time_field": "timestamp" + }, + "type": "pie" + } + ], + "title": "JDK Version Distribution" + }, + { + "formatter": "number", + "position": { + "h": 4, + "w": 3, + "x": 6, + "y": 5 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "payload.elasticsearch.cluster_health.status", + "limit": 5 + } + ], + "items": [ + { + "field": "metadata.labels.cluster_id", + "name": "a", + "statistic": "cardinality" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics" + ], + "time_field": "timestamp" + }, + "type": "pie" + } + ], + "title": "Health Status" + }, + { + "formatter": "number", + "position": { + "h": 4, + "w": 3, + "x": 0, + "y": 5 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "location.provider", + "limit": 5 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_cluster" + ] + }, + "type": "pie" + } + ], + "title": "Service Provider" + }, + { + "formatter": "number", + "position": { + "h": 4, + "w": 3, + "x": 3, + "y": 5 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "distribution", + "limit": 5 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_cluster" + ] + }, + "type": "pie" + } + ], + "title": "Search Engine Distribution" + }, + { + "data_type": "normal", + "formatter": "number", + "order": "desc", + "position": { + "h": 5, + "w": 6, + "x": 6, + "y": 14 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "metadata.labels.node_id", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.elasticsearch.node_stats.os.cpu.load_average.15m", + "name": "a", + "statistic": "max" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.name\": {\n \"value\": \"node_stats\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "column" + } + ], + "size": 10, + "title": "Node CPU Load TOP10" + }, + { + "data_type": "normal", + "formatter": "percent", + "order": "desc", + "position": { + "h": 5, + "w": 6, + "x": 0, + "y": 9 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "(b-a)/b*100", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 50 + }, + { + "field": "metadata.labels.node_name", + "limit": 500 + } + ], + "items": [ + { + "field": "payload.elasticsearch.node_stats.fs.total.available_in_bytes", + "name": "a", + "statistic": "max" + }, + { + "field": "payload.elasticsearch.node_stats.fs.total.total_in_bytes", + "name": "b", + "statistic": "max" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "column" + } + ], + "size": 10, + "title": "Node Disk Usage TOP10" + }, + { + "data_type": "timeseries", + "formatter": "percent", + "position": { + "h": 5, + "w": 6, + "x": 6, + "y": 9 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 50 + }, + { + "field": "metadata.labels.node_name", + "limit": 500 + } + ], + "items": [ + { + "field": "metadata.datatype", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "column" + } + ], + "title": "Node JVM Heap Usage TOP10" + }, + { + "data_type": "normal", + "formatter": "bytes", + "order": "desc", + "position": { + "h": 5, + "w": 6, + "x": 0, + "y": 14 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 50 + }, + { + "field": "metadata.labels.index_name", + "limit": 1000 + } + ], + "items": [ + { + "field": "payload.elasticsearch.index_stats.shard_info.store_in_bytes", + "name": "a", + "statistic": "max" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "column" + } + ], + "size": 10, + "title": "Index Shard Size TOP10" + } + ] + }, + "type": "workspace", + "is_fixed": true +} +POST $[[INDEX_PREFIX]]layout/doc/cgjpvt53q95r17vbdteg +{ + "id": "cgjpvt53q95r17vbdteg", + "created": "2023-04-01T03:07:32.165611988Z", + "updated": "2023-05-25T07:57:44.860138405Z", + "name": "Metrics&Logging Overview", + "description": "", + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + }, + "view_id": "", + "config": { + "cols": 12, + "globalQueries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": ".infini_metrics*", + "time_field": "timestamp" + }, + "global_queries": {}, + "row_height": 60, + "widgets": [ + { + "drilling": { + "new_tab_switch": false, + "url": "" + }, + "formatter": "number", + "position": { + "h": 4, + "w": 12, + "x": 0, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "payload.elasticsearch.cluster_stats.indices.docs.deleted", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "Log Collection" + }, + { + "formatter": "number", + "position": { + "h": 4, + "w": 6, + "x": 6, + "y": 8 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "agent.major_ip", + "limit": 5 + } + ], + "items": [ + { + "field": "payload.host.network_sockets.all.established", + "name": "a", + "statistic": "p99" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "TCP Connection" + }, + { + "formatter": "number", + "position": { + "h": 4, + "w": 6, + "x": 0, + "y": 12 + }, + "series": [ + { + "metric": { + "bucket_size": "30s", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "agent.major_ip", + "limit": 5 + } + ], + "items": [ + { + "field": "payload.host.network_sockets.all.connections", + "name": "a", + "statistic": "min" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "area" + } + ], + "title": "TCP Connections" + }, + { + "data_type": "timeseries", + "formatter": "number", + "position": { + "h": 4, + "w": 12, + "x": 0, + "y": 4 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "metadata.category", + "limit": 50 + }, + { + "field": "metadata.name", + "limit": 50 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "column" + } + ], + "title": "Log Data Distribution" + }, + { + "formatter": "bytes", + "position": { + "h": 4, + "w": 6, + "x": 0, + "y": 8 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "agent.major_ip", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.host.network_summary.in.bytes", + "name": "a", + "statistic": "derivative" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"host\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"network_summary\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "area" + } + ], + "title": "Network Traffic (In)" + }, + { + "formatter": "bytes", + "position": { + "h": 4, + "w": 6, + "x": 6, + "y": 12 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "agent.major_ip", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.host.network_summary.out.bytes", + "name": "a", + "statistic": "derivative" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"host\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"network_summary\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "Network Traffic (Out)" + } + ] + }, + "type": "workspace", + "is_fixed": true + } + +POST $[[INDEX_PREFIX]]layout/doc/cgjqcg53q95r17vbfo10 +{ + "id": "cgjqcg53q95r17vbfo10", + "created": "2023-04-01T03:34:24.919282378Z", + "updated": "2023-06-07T07:54:46.991959699Z", + "name": "INFINI Gateway", + "description": "", + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + }, + "view_id": "", + "config": { + "cols": 12, + "globalQueries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": ".infini_metrics*", + "time_field": "timestamp", + "time_range": { + "from": "now-30m", + "to": "now" + } + }, + "global_queries": {}, + "row_height": 60, + "widgets": [ + { + "formatter": "number", + "position": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "10s", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "metadata.labels.name", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.instance.entry.my_es_entry.open_connections", + "name": "a", + "statistic": "max" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"instance\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"gateway\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "Gateway Client Connections" + }, + { + "bucket_size": "auto", + "drilling": {}, + "formatter": "percent", + "position": { + "h": 4, + "w": 4, + "x": 4, + "y": 0 + }, + "series": [ + { + "metric": { + "formula": "a", + "groups": [ + { + "field": "metadata.labels.name", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.instance.system.cpu", + "name": "a", + "statistic": "max" + } + ], + "name": "Overall CPU" + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"instance\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"gateway\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "line" + }, + { + "metric": { + "formula": "(a + b) / c * 100", + "groups": [ + { + "field": "metadata.labels.name", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.instance.system.sys_in_ms", + "name": "a", + "statistic": "derivative" + }, + { + "field": "payload.instance.system.user_in_ms", + "name": "b", + "statistic": "derivative" + }, + { + "field": "timestamp", + "name": "c", + "statistic": "derivative" + } + ], + "name": "Real-Time CPU" + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"instance\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"gateway\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "Gateway CPU Utilization" + }, + { + "formatter": "bytes", + "position": { + "h": 4, + "w": 4, + "x": 8, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "10s", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "metadata.labels.name", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.instance.system.mem", + "name": "a", + "statistic": "max" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"instance\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"gateway\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "Gateway Memory Usage" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 5, + "w": 12, + "x": 0, + "y": 4 + }, + "series": [ + { + "metric": { + "bucket_size": "10s", + "formula": "a", + "groups": [ + { + "field": "metadata.labels.name", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.instance.system.goroutines", + "name": "a", + "statistic": "max" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"instance\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"gateway\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "Gateway Goroutine" + }, + { + "formatter": "number", + "position": { + "h": 5, + "w": 12, + "x": 0, + "y": 9 + }, + "series": [ + { + "metric": { + "bucket_size": "10s", + "format_type": "num", + "formula": "a", + "groups": [ + { + "field": "metadata.labels.name", + "limit": 50 + } + ], + "items": [ + { + "field": "payload.instance.system.objects", + "name": "a", + "statistic": "max" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "query": "{\n \"bool\": {\n \"must\": [\n {\n \"term\": {\n \"metadata.category\": {\n \"value\": \"instance\"\n }\n }\n },{\n \"term\": {\n \"metadata.name\": {\n \"value\": \"gateway\"\n }\n }\n }\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "line" + } + ], + "title": "Gateway Active Objects" + } + ] + }, + "type": "workspace", + "is_fixed": true + } +POST $[[INDEX_PREFIX]]layout/doc/ch0u5d53q95poj13629g +{ + "id": "ch0u5d53q95poj13629g", + "created": "2023-04-21T01:11:16.129965374Z", + "updated": "2023-05-25T12:23:10.993444364Z", + "name": "Clusters Overview", + "description": "", + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + }, + "view_id": "", + "config": { + "cols": 12, + "global_queries": {}, + "row_height": 60, + "widgets": [ + { + "formatter": "number", + "position": { + "h": 3, + "w": 3, + "x": 3, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "items": [ + { + "field": "agent.major_ip", + "name": "a", + "statistic": "cardinality" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "number" + } + ], + "title": "Number Of Hosts" + }, + { + "drilling": {}, + "page_size": 10, + "position": { + "h": 5, + "w": 12, + "x": 0, + "y": 4 + }, + "series": [ + { + "columns": [ + { + "display": "ClusterName", + "name": "name", + "type": "string" + }, + { + "name": "distribution", + "type": "string" + }, + { + "name": "endpoint", + "type": "string" + }, + { + "display": "Time", + "formatter": "time", + "name": "created", + "type": "date" + } + ], + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_cluster" + ] + }, + "type": "table" + } + ], + "title": "Cluster List" + }, + { + "formatter": "number", + "position": { + "h": 3, + "w": 3, + "x": 6, + "y": 1 + }, + "series": [ + { + "metric": { + "bucket_size": "1m", + "format_type": "num", + "formula": "a", + "items": [ + { + "field": "metadata.labels.node_id", + "name": "a", + "statistic": "cardinality" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "number" + } + ], + "title": "Number Of Nodes" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 3, + "w": 3, + "x": 0, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "metadata.labels.cluster_id", + "name": "a", + "statistic": "cardinality" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_metrics*" + ], + "time_field": "timestamp" + }, + "type": "number" + } + ], + "title": "Number of Clusters" + } + ] + }, + "type": "workspace", + "is_fixed": true +} +POST $[[INDEX_PREFIX]]layout/doc/cgjqcg53q95r17vbfo10 +{ + "id": "chnmaht3q95ph02nfpsg", + "created": "2023-05-25T13:40:23.821091313Z", + "updated": "2023-06-08T03:49:09.247667399Z", + "name": "Requests-analysis", + "description": "", + "creator": { + "name": "$[[USERNAME]]", + "id": "$[[USER_ID]]" + }, + "view_id": "", + "config": { + "cols": 12, + "global_queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_requests_logging*" + ], + "time_field": "timestamp" + }, + "row_height": 60, + "widgets": [ + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 2, + "w": 2, + "x": 0, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "number" + } + ], + "title": "Total Requests" + }, + { + "bucket_size": "auto", + "drilling": {}, + "formatter": "number", + "position": { + "h": 4, + "w": 12, + "x": 0, + "y": 5 + }, + "series": [ + { + "metric": { + "formula": "a", + "items": [ + { + "field": "response.elapsed", + "name": "a", + "statistic": "p99" + } + ], + "name": "P99" + }, + "queries": { + "indices": [] + }, + "type": "line" + }, + { + "metric": { + "formula": "a", + "items": [ + { + "field": "response.elapsed", + "name": "a", + "statistic": "avg" + } + ], + "name": "AVG" + }, + "queries": { + "indices": [] + }, + "type": "line" + }, + { + "metric": { + "formula": "a", + "items": [ + { + "field": "response.elapsed", + "name": "a", + "statistic": "p50" + } + ], + "name": "P50" + }, + "queries": { + "indices": [] + }, + "type": "line" + } + ], + "title": "Request Latency" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 3, + "w": 12, + "x": 0, + "y": 2 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "area" + } + ], + "title": "Request rate" + }, + { + "data_type": "timeseries", + "drilling": {}, + "formatter": "number", + "position": { + "h": 5, + "w": 12, + "x": 0, + "y": 14 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "request.path", + "limit": 5 + } + ], + "items": [ + { + "field": "response.elapsed", + "name": "a", + "statistic": "p95" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "column" + } + ], + "title": "wrong" + }, + { + "bucket_size": "auto", + "data_type": "timeseries", + "drilling": {}, + "formatter": "number", + "position": { + "h": 5, + "w": 12, + "x": 0, + "y": 9 + }, + "series": [ + { + "metric": { + "formula": "a", + "groups": [ + { + "field": "request.path", + "limit": 6 + } + ], + "items": [ + { + "field": "request.path", + "name": "a", + "statistic": "count" + } + ], + "name": "" + }, + "queries": { + "indices": [] + }, + "type": "column" + } + ], + "title": "Request rate group by path" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 2, + "w": 2, + "x": 2, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_requests_logging*" + ], + "query": "{\n \"bool\": {\n \"filter\": [\n {\"term\": {\"response.cached\": {\"value\": \"true\" } }}\n\n ]\n }\n }", + "time_field": "timestamp" + }, + "type": "number" + } + ], + "title": "Cached Requests" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 4, + "w": 2, + "x": 0, + "y": 23 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "response.status_code", + "limit": 5 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "pie" + } + ], + "title": "Response code" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 2, + "w": 2, + "x": 10, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "cluster_id": "$[[RESOURCE_ID]]", + "indices": [ + ".infini_requests_logging*" + ], + "query": "{\n \"range\": {\n \"response.elapsed\": {\n \"gte\": 1000\n }\n }\n }", + "time_field": "timestamp" + }, + "type": "number" + } + ], + "title": "Slow Queries" + }, + { + "drilling": {}, + "page_size": 10, + "position": { + "h": 5, + "w": 12, + "x": 0, + "y": 27 + }, + "series": [ + { + "columns": [ + { + "name": "flow.from", + "type": "string" + }, + { + "name": "flow.relay", + "type": "string" + }, + { + "name": "flow.to", + "type": "string" + }, + { + "name": "request.method", + "type": "string" + }, + { + "name": "request.path", + "type": "string" + }, + { + "name": "response.status_code", + "type": "number" + } + ], + "queries": { + "indices": [] + }, + "type": "table" + } + ], + "title": "Request detail" + }, + { + "data_type": "timeseries", + "drilling": {}, + "formatter": "bytes", + "position": { + "h": 4, + "w": 5, + "x": 7, + "y": 19 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "remote_ip", + "limit": 5 + } + ], + "items": [ + { + "field": "response.body_length", + "name": "a", + "statistic": "sum" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "column" + } + ], + "title": "Response size" + }, + { + "data_type": "timeseries", + "drilling": {}, + "formatter": "bytes", + "position": { + "h": 4, + "w": 5, + "x": 2, + "y": 19 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "remote_ip", + "limit": 5 + } + ], + "items": [ + { + "field": "request.body_length", + "name": "a", + "statistic": "sum" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "column" + } + ], + "title": "Request size" + }, + { + "data_type": "timeseries", + "drilling": {}, + "formatter": "number", + "position": { + "h": 4, + "w": 5, + "x": 2, + "y": 23 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "flow.to", + "limit": 5 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "column" + } + ], + "title": "Request upstreams" + }, + { + "data_type": "timeseries", + "drilling": {}, + "formatter": "number", + "position": { + "h": 4, + "w": 5, + "x": 7, + "y": 23 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "request.method", + "limit": 5 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "column" + } + ], + "title": "Request methods" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 4, + "w": 2, + "x": 0, + "y": 19 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "groups": [ + { + "field": "flow.from", + "limit": 5 + } + ], + "items": [ + { + "field": "*", + "name": "a", + "statistic": "count" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "pie" + } + ], + "title": "Clients ip" + }, + { + "drilling": {}, + "formatter": "bytes", + "position": { + "h": 2, + "w": 2, + "x": 8, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "response.body_length", + "name": "a", + "statistic": "sum" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "number" + } + ], + "title": "Response traffic" + }, + { + "drilling": {}, + "formatter": "bytes", + "position": { + "h": 2, + "w": 2, + "x": 6, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "request.body_length", + "name": "a", + "statistic": "sum" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "number" + } + ], + "title": "Request traffic" + }, + { + "drilling": {}, + "formatter": "number", + "position": { + "h": 2, + "w": 2, + "x": 4, + "y": 0 + }, + "series": [ + { + "metric": { + "bucket_size": "auto", + "formula": "a", + "items": [ + { + "field": "remote_ip", + "name": "a", + "statistic": "cardinality" + } + ] + }, + "queries": { + "indices": [] + }, + "type": "number" + } + ], + "title": "Clients" + } + ] + }, + "type": "workspace", + "is_fixed": true +} GET / From be1dd920c7e44d0e85ab87ede350fcce3b05f44c Mon Sep 17 00:00:00 2001 From: liugq Date: Thu, 8 Jun 2023 16:31:32 +0800 Subject: [PATCH 6/7] set default agent setup config --- console.yml | 26 ++++++++--------- modules/agent/agent.go | 16 ++++++++--- modules/agent/api/setup.go | 20 +++++++++---- modules/agent/common/cert.go | 6 ---- modules/agent/common/config.go | 51 ++++++++++++++++++++++++++++++++-- modules/agent/common/helper.go | 15 +++++++++- 6 files changed, 101 insertions(+), 33 deletions(-) diff --git a/console.yml b/console.yml index 7b80fc8b..0a4588af 100644 --- a/console.yml +++ b/console.yml @@ -1,10 +1,10 @@ path.configs: "config" configs.auto_reload: true -env: - INFINI_CONSOLE_ENDPOINT: "http://127.0.0.1:9000" - INGEST_CLUSTER_ENDPOINT: "https://127.0.0.1:9200" - INGEST_CLUSTER_CREDENTIAL_ID: chjkp9dath21f1ae9tq0 +#env: +# INFINI_CONSOLE_ENDPOINT: "http://127.0.0.1:9000" +# INGEST_CLUSTER_ENDPOINT: "https://127.0.0.1:9200" +# INGEST_CLUSTER_CREDENTIAL_ID: chjkp9dath21f1ae9tq0 web: enabled: true @@ -71,12 +71,12 @@ badger: # redirect_url: "" # scopes: [] -agent: - setup: - download_url: "https://release.infinilabs.com/agent/stable" - version: 0.5.0-214 - ca_cert: "config/certs/ca.crt" - ca_key: "config/certs/ca.key" - console_endpoint: $[[env.INFINI_CONSOLE_ENDPOINT]] - ingest_cluster_endpoint: $[[env.INGEST_CLUSTER_ENDPOINT]] - ingest_cluster_credential_id: $[[env.INGEST_CLUSTER_CREDENTIAL_ID]] \ No newline at end of file +#agent: +# setup: +# download_url: "https://release.infinilabs.com/agent/stable" +# version: 0.5.0-214 +# ca_cert: "config/certs/ca.crt" +# ca_key: "config/certs/ca.key" +# console_endpoint: $[[env.INFINI_CONSOLE_ENDPOINT]] +# ingest_cluster_endpoint: $[[env.INGEST_CLUSTER_ENDPOINT]] +# ingest_cluster_credential_id: $[[env.INGEST_CLUSTER_CREDENTIAL_ID]] \ No newline at end of file diff --git a/modules/agent/agent.go b/modules/agent/agent.go index 1a9241e4..9b86e2d9 100644 --- a/modules/agent/agent.go +++ b/modules/agent/agent.go @@ -49,15 +49,23 @@ func (module *AgentModule) Start() error { var ( executor client.Executor err error + caFile string + caKey string ) - if module.AgentConfig.Setup == nil { - executor = &client.HttpExecutor{} - }else{ - executor, err = client.NewMTLSExecutor(module.AgentConfig.Setup.CACertFile, module.AgentConfig.Setup.CAKeyFile) + if module.AgentConfig.Setup != nil { + caFile = module.AgentConfig.Setup.CACertFile + caKey = module.AgentConfig.Setup.CAKeyFile + } + if caFile == "" && caKey == "" { + caFile, caKey, err = common.GetOrInitDefaultCaCerts() if err != nil { panic(err) } } + executor, err = client.NewMTLSExecutor(caFile, caKey) + if err != nil { + panic(err) + } agClient := &client.Client{ Executor: executor, } diff --git a/modules/agent/api/setup.go b/modules/agent/api/setup.go index 5a4ff435..c63017c9 100644 --- a/modules/agent/api/setup.go +++ b/modules/agent/api/setup.go @@ -74,11 +74,7 @@ func (h *APIHandler) generateInstallCommand(w http.ResponseWriter, req *http.Req tokens.Store(tokenStr, t) consoleEndpoint := agCfg.Setup.ConsoleEndpoint if consoleEndpoint == "" { - scheme := "http" - if req.TLS != nil { - scheme = "https" - } - consoleEndpoint = fmt.Sprintf("%s://%s", scheme, req.Host) + consoleEndpoint = getDefaultConsoleEndpoint(req) } h.WriteJSON(w, util.MapStr{ "script": fmt.Sprintf(`sudo BASE_URL="%s" AGENT_VER="%s" INSTALL_PATH="/opt" bash -c "$(curl -L '%s/agent/install.sh?token=%s')"`, agCfg.Setup.DownloadURL, agCfg.Setup.Version, consoleEndpoint, tokenStr), @@ -87,6 +83,14 @@ func (h *APIHandler) generateInstallCommand(w http.ResponseWriter, req *http.Req }, http.StatusOK) } +func getDefaultConsoleEndpoint(req *http.Request) string{ + scheme := "http" + if req.TLS != nil { + scheme = "https" + } + return fmt.Sprintf("%s://%s", scheme, req.Host) +} + func (h *APIHandler) getInstallScript(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { tokenStr := h.GetParameter(req, "token") if strings.TrimSpace(tokenStr) == "" { @@ -127,10 +131,14 @@ func (h *APIHandler) getInstallScript(w http.ResponseWriter, req *http.Request, if port == "" { port = "8080" } + consoleEndpoint := agCfg.Setup.ConsoleEndpoint + if consoleEndpoint == "" { + consoleEndpoint = getDefaultConsoleEndpoint(req) + } _, err = tpl.Execute(w, map[string]interface{}{ "base_url": agCfg.Setup.DownloadURL, "agent_version": agCfg.Setup.Version, - "console_endpoint": agCfg.Setup.ConsoleEndpoint, + "console_endpoint": consoleEndpoint, "client_crt": clientCertPEM, "client_key": clientKeyPEM, "ca_crt": caCert, diff --git a/modules/agent/common/cert.go b/modules/agent/common/cert.go index 61521509..9b324009 100644 --- a/modules/agent/common/cert.go +++ b/modules/agent/common/cert.go @@ -24,9 +24,6 @@ func GenerateServerCert(caFile, caKey string) (caCert, serverCertPEM, serverKeyP func generateCert(caFile, caKey string, isServer bool)(caCert, instanceCertPEM, instanceKeyPEM []byte, err error){ pool := x509.NewCertPool() - if caFile == "" { - caFile = path.Join(global.Env().GetConfigDir(), "certs", "ca.crt") - } caCert, err = os.ReadFile(caFile) if err != nil { return @@ -39,9 +36,6 @@ func generateCert(caFile, caKey string, isServer bool)(caCert, instanceCertPEM, if err != nil { return } - if caKey == "" { - caKey = path.Join(global.Env().GetConfigDir(), "certs", "ca.key") - } var keyBytes []byte keyBytes, err = os.ReadFile(caKey) if err != nil { diff --git a/modules/agent/common/config.go b/modules/agent/common/config.go index 33319101..54b1cc0e 100644 --- a/modules/agent/common/config.go +++ b/modules/agent/common/config.go @@ -5,17 +5,62 @@ package common import ( + "crypto/x509" + "encoding/pem" + log "github.com/cihub/seelog" "infini.sh/console/modules/agent/model" "infini.sh/framework/core/env" - log "src/github.com/cihub/seelog" + "infini.sh/framework/core/global" + "infini.sh/framework/core/util" + "os" + "path" ) func GetAgentConfig() *model.AgentConfig { - agentCfg := &model.AgentConfig{} + agentCfg := &model.AgentConfig{ + Enabled: true, + Setup: &model.SetupConfig{ + DownloadURL: "https://release.infinilabs.com/agent/stable", + Version: "0.5.0-214", + }, + } _, err := env.ParseConfig("agent", agentCfg ) if err != nil { - log.Error("agent config not found: %v", err) + log.Debug("agent config not found: %v", err) + } + if agentCfg.Setup.CACertFile == "" && agentCfg.Setup.CAKeyFile == "" { + agentCfg.Setup.CACertFile, agentCfg.Setup.CAKeyFile, err = GetOrInitDefaultCaCerts() + if err != nil { + log.Errorf("generate default ca certs error: %v", err) + } } return agentCfg +} + +func GetOrInitDefaultCaCerts()(string, string, error){ + dataDir := global.Env().GetDataDir() + caFile := path.Join(dataDir, "certs/ca.crt") + caKey := path.Join(dataDir, "certs/ca.key") + if !(util.FileExists(caFile) && util.FileExists(caKey) ) { + err := os.MkdirAll(path.Join(dataDir, "certs"), 0775) + if err != nil { + return "", "", err + } + log.Info("auto generating cert files") + _, rootKey, rootCertPEM := util.GetRootCert() + + caKeyPEM := pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(rootKey), + }) + _, err = util.FilePutContentWithByte(caKey, caKeyPEM) + if err != nil { + return "", "", err + } + _, err = util.FilePutContentWithByte(caFile, rootCertPEM) + if err != nil { + return "", "", err + } + } + return caFile, caKey, nil } \ No newline at end of file diff --git a/modules/agent/common/helper.go b/modules/agent/common/helper.go index 4c4b8d8a..a6d26216 100644 --- a/modules/agent/common/helper.go +++ b/modules/agent/common/helper.go @@ -11,6 +11,7 @@ import ( "infini.sh/framework/core/credential" "infini.sh/framework/core/elastic" "infini.sh/framework/core/event" + "infini.sh/framework/core/global" "infini.sh/framework/core/orm" "infini.sh/framework/core/util" log "src/github.com/cihub/seelog" @@ -385,11 +386,20 @@ func GetAgentIngestConfig() (string, *elastic.BasicAuth, error) { endpoint string ok bool ) + emptyIngestClusterEndpoint := false + if agCfg.Setup.IngestClusterEndpoint == nil { + emptyIngestClusterEndpoint = true + } if endpoint, ok = agCfg.Setup.IngestClusterEndpoint.(string);ok { if endpoint = strings.TrimSpace(endpoint); endpoint == "" { - return "", nil, fmt.Errorf("config ingest_cluster_endpoint must not be empty") + emptyIngestClusterEndpoint = true } } + if emptyIngestClusterEndpoint { + cfg := elastic.GetConfig(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) + endpoint = cfg.Endpoint + } + var ( basicAuth elastic.BasicAuth ) @@ -407,6 +417,9 @@ func GetAgentIngestConfig() (string, *elastic.BasicAuth, error) { if basicAuth, ok = info.(elastic.BasicAuth); !ok { log.Debug("invalid credential: ", cred) } + }else{ + cfg := elastic.GetConfig(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) + basicAuth = *cfg.BasicAuth } tpl := `elasticsearch: - name: default From 365dd690abbf75740c3797653709d989ff45b0a1 Mon Sep 17 00:00:00 2001 From: liugq Date: Thu, 8 Jun 2023 20:31:42 +0800 Subject: [PATCH 7/7] validate node publish address with agent instance ip list --- modules/agent/api/instance.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/modules/agent/api/instance.go b/modules/agent/api/instance.go index 0495125c..4a9a92fb 100644 --- a/modules/agent/api/instance.go +++ b/modules/agent/api/instance.go @@ -21,6 +21,7 @@ import ( "infini.sh/framework/core/util" elastic2 "infini.sh/framework/modules/elastic" "infini.sh/framework/modules/elastic/common" + "net" "net/http" "strconv" "time" @@ -491,6 +492,16 @@ func (h *APIHandler) authESNode(w http.ResponseWriter, req *http.Request, ps htt h.WriteError(w, err.Error(), http.StatusInternalServerError) return } + host, _, err := net.SplitHostPort(nodeInfo.PublishAddress) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if !util.StringInArray(inst.IPS, host) { + h.WriteError(w, fmt.Sprintf("got node host %s not match any ip of %v", host, inst.IPS), http.StatusInternalServerError) + return + } nodeInfo.ID = oldNodeInfo.ID nodeInfo.AgentID = inst.ID err = orm.Save(nil, nodeInfo)